{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2051, 60000, 28, 28)\n",
      "47040000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "IOPub data rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_data_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_data_rate_limit=1000000.0 (bytes/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import struct\n",
    "with open('./MNIST_data/train-images-idx3-ubyte','rb') as f:\n",
    "    buffer = f.read(4*4) # 4个int\n",
    "    head = struct.unpack('>iiii',buffer)\n",
    "    print(head)\n",
    "    length = head[1] * head[2] * head[3]\n",
    "    print(length)\n",
    "    buffer = f.read(length)\n",
    "    print(buffer)\n",
    "    data = struct.unpack('>{}B'.format(length),buffer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "47040000"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tuple"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "imgs = np.reshape(data,(head[1],head[2],head[3]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(60000, 28, 28)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "imgs.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAN80lEQVR4nO3df6hcdXrH8c+ncf3DrBpTMYasNhuRWBWbLRqLSl2RrD9QNOqWDVgsBrN/GHChhEr6xyolEuqP0qAsuYu6sWyzLqgYZVkVo6ZFCF5j1JjU1YrdjV6SSozG+KtJnv5xT+Su3vnOzcyZOZP7vF9wmZnzzJnzcLife87Md879OiIEYPL7k6YbANAfhB1IgrADSRB2IAnCDiRxRD83ZpuP/oEeiwiPt7yrI7vtS22/aftt27d281oAesudjrPbniLpd5IWSNou6SVJiyJia2EdjuxAj/XiyD5f0tsR8U5EfCnpV5Ku6uL1APRQN2GfJekPYx5vr5b9EdtLbA/bHu5iWwC61M0HdOOdKnzjND0ihiQNSZzGA03q5si+XdJJYx5/R9L73bUDoFe6CftLkk61/V3bR0r6kaR19bQFoG4dn8ZHxD7bSyU9JWmKpAci4o3aOgNQq46H3jraGO/ZgZ7ryZdqABw+CDuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUii4ymbcXiYMmVKsX7sscf2dPtLly5tWTvqqKOK686dO7dYv/nmm4v1u+66q2Vt0aJFxXU///zzYn3lypXF+u23316sN6GrsNt+V9IeSfsl7YuIs+toCkD96jiyXxQRH9TwOgB6iPfsQBLdhj0kPW37ZdtLxnuC7SW2h20Pd7ktAF3o9jT+/Ih43/YJkp6x/V8RsWHsEyJiSNKQJNmOLrcHoENdHdkj4v3qdqekxyTNr6MpAPXrOOy2p9o++uB9ST+QtKWuxgDUq5vT+BmSHrN98HX+PSJ+W0tXk8zJJ59crB955JHF+nnnnVesX3DBBS1r06ZNK6577bXXFutN2r59e7G+atWqYn3hwoUta3v27Cmu++qrrxbrL7zwQrE+iDoOe0S8I+kvauwFQA8x9AYkQdiBJAg7kARhB5Ig7EASjujfl9om6zfo5s2bV6yvX7++WO/1ZaaD6sCBA8X6jTfeWKx/8sknHW97ZGSkWP/www+L9TfffLPjbfdaRHi85RzZgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJxtlrMH369GJ948aNxfqcOXPqbKdW7XrfvXt3sX7RRRe1rH355ZfFdbN+/6BbjLMDyRF2IAnCDiRB2IEkCDuQBGEHkiDsQBJM2VyDXbt2FevLli0r1q+44opi/ZVXXinW2/1L5ZLNmzcX6wsWLCjW9+7dW6yfccYZLWu33HJLcV3UiyM7kARhB5Ig7EAShB1IgrADSRB2IAnCDiTB9ewD4JhjjinW200vvHr16pa1xYsXF9e9/vrri/W1a9cW6xg8HV/PbvsB2zttbxmzbLrtZ2y/Vd0eV2ezAOo3kdP4X0i69GvLbpX0bEScKunZ6jGAAdY27BGxQdLXvw96laQ11f01kq6uuS8ANev0u/EzImJEkiJixPYJrZ5oe4mkJR1uB0BNen4hTEQMSRqS+IAOaFKnQ287bM+UpOp2Z30tAeiFTsO+TtIN1f0bJD1eTzsAeqXtabzttZK+L+l429sl/VTSSkm/tr1Y0u8l/bCXTU52H3/8cVfrf/TRRx2ve9NNNxXrDz/8cLHebo51DI62YY+IRS1KF9fcC4Ae4uuyQBKEHUiCsANJEHYgCcIOJMElrpPA1KlTW9aeeOKJ4roXXnhhsX7ZZZcV608//XSxjv5jymYgOcIOJEHYgSQIO5AEYQeSIOxAEoQdSIJx9knulFNOKdY3bdpUrO/evbtYf+6554r14eHhlrX77ruvuG4/fzcnE8bZgeQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJxtmTW7hwYbH+4IMPFutHH310x9tevnx5sf7QQw8V6yMjIx1vezJjnB1IjrADSRB2IAnCDiRB2IEkCDuQBGEHkmCcHUVnnnlmsX7PPfcU6xdf3Plkv6tXry7WV6xYUay/9957HW/7cNbxOLvtB2zvtL1lzLLbbL9ne3P1c3mdzQKo30RO438h6dJxlv9LRMyrfn5Tb1sA6tY27BGxQdKuPvQCoIe6+YBuqe3XqtP841o9yfYS28O2W/8zMgA912nYfybpFEnzJI1IurvVEyNiKCLOjoizO9wWgBp0FPaI2BER+yPigKSfS5pfb1sA6tZR2G3PHPNwoaQtrZ4LYDC0HWe3vVbS9yUdL2mHpJ9Wj+dJCknvSvpxRLS9uJhx9sln2rRpxfqVV17ZstbuWnl73OHir6xfv75YX7BgQbE+WbUaZz9iAisuGmfx/V13BKCv+LoskARhB5Ig7EAShB1IgrADSXCJKxrzxRdfFOtHHFEeLNq3b1+xfskll7SsPf/888V1D2f8K2kgOcIOJEHYgSQIO5AEYQeSIOxAEoQdSKLtVW/I7ayzzirWr7vuumL9nHPOaVlrN47eztatW4v1DRs2dPX6kw1HdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgnH2SW7u3LnF+tKlS4v1a665plg/8cQTD7mnidq/f3+xPjJS/u/lBw4cqLOdwx5HdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgnH2w0C7sexFi8abaHdUu3H02bNnd9JSLYaHh4v1FStWFOvr1q2rs51Jr+2R3fZJtp+zvc32G7ZvqZZPt/2M7beq2+N63y6ATk3kNH6fpL+PiD+X9FeSbrZ9uqRbJT0bEadKerZ6DGBAtQ17RIxExKbq/h5J2yTNknSVpDXV09ZIurpXTQLo3iG9Z7c9W9L3JG2UNCMiRqTRPwi2T2ixzhJJS7prE0C3Jhx229+W9Iikn0TEx/a4c8d9Q0QMSRqqXoOJHYGGTGjozfa3NBr0X0bEo9XiHbZnVvWZknb2pkUAdWh7ZPfoIfx+Sdsi4p4xpXWSbpC0srp9vCcdTgIzZswo1k8//fRi/d577y3WTzvttEPuqS4bN24s1u+8886WtccfL//KcIlqvSZyGn++pL+V9LrtzdWy5RoN+a9tL5b0e0k/7E2LAOrQNuwR8Z+SWr1Bv7jedgD0Cl+XBZIg7EAShB1IgrADSRB2IAkucZ2g6dOnt6ytXr26uO68efOK9Tlz5nTUUx1efPHFYv3uu+8u1p966qli/bPPPjvkntAbHNmBJAg7kARhB5Ig7EAShB1IgrADSRB2IIk04+znnntusb5s2bJiff78+S1rs2bN6qinunz66acta6tWrSque8cddxTre/fu7agnDB6O7EAShB1IgrADSRB2IAnCDiRB2IEkCDuQRJpx9oULF3ZV78bWrVuL9SeffLJY37dvX7FeuuZ89+7dxXWRB0d2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUjCEVF+gn2SpIcknSjpgKShiPhX27dJuknS/1ZPXR4Rv2nzWuWNAehaRIw76/JEwj5T0syI2GT7aEkvS7pa0t9I+iQi7ppoE4Qd6L1WYZ/I/Owjkkaq+3tsb5PU7L9mAXDIDuk9u+3Zkr4naWO1aKnt12w/YPu4FusssT1se7irTgF0pe1p/FdPtL8t6QVJKyLiUdszJH0gKST9k0ZP9W9s8xqcxgM91vF7dkmy/S1JT0p6KiLuGac+W9KTEXFmm9ch7ECPtQp729N425Z0v6RtY4NefXB30EJJW7ptEkDvTOTT+Ask/Yek1zU69CZJyyUtkjRPo6fx70r6cfVhXum1OLIDPdbVaXxdCDvQex2fxgOYHAg7kARhB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCsANJ9HvK5g8k/c+Yx8dXywbRoPY2qH1J9NapOnv7s1aFvl7P/o2N28MRcXZjDRQMam+D2pdEb53qV2+cxgNJEHYgiabDPtTw9ksGtbdB7Uuit071pbdG37MD6J+mj+wA+oSwA0k0Enbbl9p+0/bbtm9toodWbL9r+3Xbm5uen66aQ2+n7S1jlk23/Yztt6rbcefYa6i322y/V+27zbYvb6i3k2w/Z3ub7Tds31Itb3TfFfrqy37r+3t221Mk/U7SAknbJb0kaVFEbO1rIy3YflfS2RHR+BcwbP+1pE8kPXRwai3b/yxpV0SsrP5QHhcR/zAgvd2mQ5zGu0e9tZpm/O/U4L6rc/rzTjRxZJ8v6e2IeCcivpT0K0lXNdDHwIuIDZJ2fW3xVZLWVPfXaPSXpe9a9DYQImIkIjZV9/dIOjjNeKP7rtBXXzQR9lmS/jDm8XYN1nzvIelp2y/bXtJ0M+OYcXCarer2hIb7+bq203j309emGR+YfdfJ9OfdaiLs401NM0jjf+dHxF9KukzSzdXpKibmZ5JO0egcgCOS7m6ymWqa8Uck/SQiPm6yl7HG6asv+62JsG+XdNKYx9+R9H4DfYwrIt6vbndKekyjbzsGyY6DM+hWtzsb7ucrEbEjIvZHxAFJP1eD+66aZvwRSb+MiEerxY3vu/H66td+ayLsL0k61fZ3bR8p6UeS1jXQxzfYnlp9cCLbUyX9QIM3FfU6STdU92+Q9HiDvfyRQZnGu9U042p43zU+/XlE9P1H0uUa/UT+vyX9YxM9tOhrjqRXq583mu5N0lqNntb9n0bPiBZL+lNJz0p6q7qdPkC9/ZtGp/Z+TaPBmtlQbxdo9K3ha5I2Vz+XN73vCn31Zb/xdVkgCb5BByRB2IEkCDuQBGEHkiDsQBKEHUiCsANJ/D+f1mbtgJ8kQQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAOEUlEQVR4nO3dcYwV5bnH8d8jLUalENSIG9Ha22Bym0YXQUJiU6lNG4sm0JhWiHFp2mRJLAk1jam2q5DUGxujNGoicaukWLlCFS3Y1EsNS/TemDSuSBVLW6mhdMuGFTWyxEQqPPePHZoVd95Zzpk5c+D5fpLNOWeenTOPx/0xc847c15zdwE49Z1WdwMAWoOwA0EQdiAIwg4EQdiBID7Vyo2ZGR/9AxVzdxtreVN7djO7xsz+Yma7zey2Zp4LQLWs0XF2M5sg6a+SviZpQNLLkha7+58S67BnBypWxZ59jqTd7v6Wux+WtF7SgiaeD0CFmgn7BZL+MerxQLbsY8ys28z6zay/iW0BaFIzH9CNdajwicN0d++V1CtxGA/UqZk9+4CkC0c9ni5pX3PtAKhKM2F/WdIMM/ucmU2UtEjS5nLaAlC2hg/j3f0jM1smaYukCZLWuPsbpXUGoFQND701tDHeswOVq+SkGgAnD8IOBEHYgSAIOxAEYQeCIOxAEIQdCIKwA0EQdiAIwg4EQdiBIAg7EARhB4Ig7EAQhB0IgrADQRB2IAjCDgRB2IEgCDsQBGEHgmjplM049cyaNStZX7ZsWW6tq6srue5jjz2WrD/44IPJ+vbt25P1aNizA0EQdiAIwg4EQdiBIAg7EARhB4Ig7EAQzOKKpM7OzmS9r68vWZ88eXKZ7XzM+++/n6yfc845lW27neXN4trUSTVmtkfSsKQjkj5y99nNPB+A6pRxBt1X3P1ACc8DoEK8ZweCaDbsLun3ZvaKmXWP9Qtm1m1m/WbW3+S2ADSh2cP4K919n5mdJ+l5M/uzu784+hfcvVdSr8QHdECdmtqzu/u+7HZI0jOS5pTRFIDyNRx2MzvLzD5z7L6kr0vaWVZjAMrVzGH8NEnPmNmx5/lvd/+fUrpCy8yZkz4Y27hxY7I+ZcqUZD11Hsfw8HBy3cOHDyfrRePoc+fOza0VXetetO2TUcNhd/e3JF1WYi8AKsTQGxAEYQeCIOxAEIQdCIKwA0Fwiesp4Mwzz8ytXX755cl1H3/88WR9+vTpyXo29Jor9fdVNPx1zz33JOvr169P1lO99fT0JNe9++67k/V2lneJK3t2IAjCDgRB2IEgCDsQBGEHgiDsQBCEHQiCKZtPAQ8//HBubfHixS3s5MQUnQMwadKkZP2FF15I1ufNm5dbu/TSS5PrnorYswNBEHYgCMIOBEHYgSAIOxAEYQeCIOxAEIyznwRmzZqVrF977bW5taLrzYsUjWU/++yzyfq9996bW9u3b19y3VdffTVZf++995L1q6++OrfW7OtyMmLPDgRB2IEgCDsQBGEHgiDsQBCEHQiCsANB8L3xbaCzszNZ7+vrS9YnT57c8Lafe+65ZL3oevirrroqWU9dN/7II48k13377beT9SJHjhzJrX3wwQfJdYv+u4q+875ODX9vvJmtMbMhM9s5atnZZva8mb2Z3U4ts1kA5RvPYfwvJV1z3LLbJG119xmStmaPAbSxwrC7+4uS3j1u8QJJa7P7ayUtLLkvACVr9Nz4ae4+KEnuPmhm5+X9opl1S+pucDsASlL5hTDu3iupV+IDOqBOjQ697TezDknKbofKawlAFRoN+2ZJS7L7SyRtKqcdAFUpHGc3syckzZN0rqT9klZI+o2kX0u6SNJeSd9y9+M/xBvruUIexl9yySXJ+ooVK5L1RYsWJesHDhzIrQ0ODibXveuuu5L1p556KllvZ6lx9qK/+w0bNiTrN954Y0M9tULeOHvhe3Z3zzur4qtNdQSgpThdFgiCsANBEHYgCMIOBEHYgSD4KukSnH766cl66uuUJWn+/PnJ+vDwcLLe1dWVW+vv70+ue8YZZyTrUV100UV1t1A69uxAEIQdCIKwA0EQdiAIwg4EQdiBIAg7EATj7CWYOXNmsl40jl5kwYIFyXrRtMqAxJ4dCIOwA0EQdiAIwg4EQdiBIAg7EARhB4JgnL0Eq1atStbNxvxm338rGidnHL0xp52Wvy87evRoCztpD+zZgSAIOxAEYQeCIOxAEIQdCIKwA0EQdiAIxtnH6brrrsutdXZ2Jtctmh548+bNDfWEtNRYetH/kx07dpTdTu0K9+xmtsbMhsxs56hlK83sn2a2I/tp7tsZAFRuPIfxv5R0zRjLf+7undnP78ptC0DZCsPu7i9KercFvQCoUDMf0C0zs9eyw/ypeb9kZt1m1m9m6UnHAFSq0bCvlvR5SZ2SBiXdl/eL7t7r7rPdfXaD2wJQgobC7u773f2Iux+V9AtJc8ptC0DZGgq7mXWMevhNSTvzfhdAeygcZzezJyTNk3SumQ1IWiFpnpl1SnJJeyQtrbDHtpCax3zixInJdYeGhpL1DRs2NNTTqa5o3vuVK1c2/Nx9fX3J+u23397wc7erwrC7++IxFj9aQS8AKsTpskAQhB0IgrADQRB2IAjCDgTBJa4t8OGHHybrg4ODLeqkvRQNrfX09CTrt956a7I+MDCQW7vvvtyTPiVJhw4dStZPRuzZgSAIOxAEYQeCIOxAEIQdCIKwA0EQdiAIxtlbIPJXRae+ZrtonPyGG25I1jdt2pSsX3/99cl6NOzZgSAIOxAEYQeCIOxAEIQdCIKwA0EQdiAIxtnHycwaqknSwoULk/Xly5c31FM7uOWWW5L1O+64I7c2ZcqU5Lrr1q1L1ru6upJ1fBx7diAIwg4EQdiBIAg7EARhB4Ig7EAQhB0IgnH2cXL3hmqSdP755yfrDzzwQLK+Zs2aZP2dd97Jrc2dOze57k033ZSsX3bZZcn69OnTk/W9e/fm1rZs2ZJc96GHHkrWcWIK9+xmdqGZbTOzXWb2hpktz5afbWbPm9mb2e3U6tsF0KjxHMZ/JOmH7v6fkuZK+r6ZfUHSbZK2uvsMSVuzxwDaVGHY3X3Q3bdn94cl7ZJ0gaQFktZmv7ZWUvqcUAC1OqH37GZ2saSZkv4gaZq7D0oj/yCY2Xk563RL6m6uTQDNGnfYzWySpI2SfuDuB4su/jjG3Xsl9WbPkf4kC0BlxjX0Zmaf1kjQ17n709ni/WbWkdU7JA1V0yKAMhTu2W1kF/6opF3uvmpUabOkJZJ+lt2mv9c3sAkTJiTrN998c7Je9JXIBw8ezK3NmDEjuW6zXnrppWR927ZtubU777yz7HaQMJ7D+Csl3STpdTPbkS37sUZC/msz+56kvZK+VU2LAMpQGHZ3/z9JeW/Qv1puOwCqwumyQBCEHQiCsANBEHYgCMIOBGFFl2eWurGT+Ay61KWcTz75ZHLdK664oqltF52t2Mz/w9TlsZK0fv36ZP1k/hrsU5W7j/kHw54dCIKwA0EQdiAIwg4EQdiBIAg7EARhB4JgnL0EHR0dyfrSpUuT9Z6enmS9mXH2+++/P7nu6tWrk/Xdu3cn62g/jLMDwRF2IAjCDgRB2IEgCDsQBGEHgiDsQBCMswOnGMbZgeAIOxAEYQeCIOxAEIQdCIKwA0EQdiCIwrCb2YVmts3MdpnZG2a2PFu+0sz+aWY7sp/51bcLoFGFJ9WYWYekDnffbmafkfSKpIWSvi3pkLvfO+6NcVINULm8k2rGMz/7oKTB7P6wme2SdEG57QGo2gm9ZzeziyXNlPSHbNEyM3vNzNaY2dScdbrNrN/M+pvqFEBTxn1uvJlNkvSCpP9y96fNbJqkA5Jc0k81cqj/3YLn4DAeqFjeYfy4wm5mn5b0W0lb3H3VGPWLJf3W3b9Y8DyEHahYwxfC2MhXmz4qadfooGcf3B3zTUk7m20SQHXG82n8lyT9r6TXJR3NFv9Y0mJJnRo5jN8jaWn2YV7qudizAxVr6jC+LIQdqB7XswPBEXYgCMIOBEHYgSAIOxAEYQeCIOxAEIQdCIKwA0EQdiAIwg4EQdiBIAg7EARhB4Io/MLJkh2Q9PdRj8/NlrWjdu2tXfuS6K1RZfb22bxCS69n/8TGzfrdfXZtDSS0a2/t2pdEb41qVW8cxgNBEHYgiLrD3lvz9lPatbd27Uuit0a1pLda37MDaJ269+wAWoSwA0HUEnYzu8bM/mJmu83stjp6yGNme8zs9Wwa6lrnp8vm0Bsys52jlp1tZs+b2ZvZ7Zhz7NXUW1tM452YZrzW167u6c9b/p7dzCZI+qukr0kakPSypMXu/qeWNpLDzPZImu3utZ+AYWZflnRI0mPHptYys3skvevuP8v+oZzq7j9qk95W6gSn8a6ot7xpxr+jGl+7Mqc/b0Qde/Y5kna7+1vufljSekkLauij7bn7i5LePW7xAklrs/trNfLH0nI5vbUFdx909+3Z/WFJx6YZr/W1S/TVEnWE/QJJ/xj1eEDtNd+7S/q9mb1iZt11NzOGacem2cpuz6u5n+MVTuPdSsdNM942r10j0583q46wjzU1TTuN/13p7pdL+oak72eHqxif1ZI+r5E5AAcl3VdnM9k04xsl/cDdD9bZy2hj9NWS162OsA9IunDU4+mS9tXQx5jcfV92OyTpGY287Wgn+4/NoJvdDtXcz7+5+353P+LuRyX9QjW+dtk04xslrXP3p7PFtb92Y/XVqtetjrC/LGmGmX3OzCZKWiRpcw19fIKZnZV9cCIzO0vS19V+U1FvlrQku79E0qYae/mYdpnGO2+acdX82tU+/bm7t/xH0nyNfCL/N0k/qaOHnL7+Q9Ifs5836u5N0hMaOaz7l0aOiL4n6RxJWyW9md2e3Ua9/UojU3u/ppFgddTU25c08tbwNUk7sp/5db92ib5a8rpxuiwQBGfQAUEQdiAIwg4EQdiBIAg7EARhB4Ig7EAQ/w8+sGPVrnT8WgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAM6klEQVR4nO3db4hd9Z3H8c8n2oDYKol/0sEETUuUliXqEmXVolliQzZPYh9YGrRmqTiCFVrYByv2QQVZ0MW29ImFqUrSNWspxNFQam0IRVvQMBNJNcmYxIYYJxmSFZGmKHaj330wZ7pjnHvu5N5z7rkz3/cLLvfe873nni+HfPI755575+eIEID5b0HTDQDoDcIOJEHYgSQIO5AEYQeSOLeXG7PNR/9AzSLCMy3vamS3vc72Adtv2X6gm/cCUC93ep3d9jmSDkr6uqRxSSOSNkbE/pJ1GNmBmtUxsl8v6a2IOBwRf5P0S0kbung/ADXqJuyXSXpn2vPxYtmn2B60PWp7tIttAehSNx/QzXSo8JnD9IgYkjQkcRgPNKmbkX1c0rJpz5dKOt5dOwDq0k3YRyStsL3c9kJJ35K0vZq2AFSt48P4iDht+35JL0o6R9JTEbGvss4AVKrjS28dbYxzdqB2tXypBsDcQdiBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EASHU/ZDPS7NWvWtKxt3bq1dN1bbrmltH7gwIGOempSV2G3fUTSKUkfSzodEauqaApA9aoY2f85It6t4H0A1IhzdiCJbsMekn5ne7ftwZleYHvQ9qjt0S63BaAL3R7G3xQRx21fKmmH7Tcj4uXpL4iIIUlDkmQ7utwegA51NbJHxPHi/qSkYUnXV9EUgOp1HHbb59v+wtRjSWsl7a2qMQDV6uYwfomkYdtT7/PfEfHbSrqqwc0331xav+iii0rrw8PDVbaDHrjuuuta1kZGRnrYSX/oOOwRcVjS1RX2AqBGXHoDkiDsQBKEHUiCsANJEHYgiTQ/cV29enVpfcWKFaV1Lr31nwULyseq5cuXt6xdfvnlpesWl5TnFUZ2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUgizXX2u+66q7T+yiuv9KgTVGVgYKC0fs8997SsPf3006Xrvvnmmx311M8Y2YEkCDuQBGEHkiDsQBKEHUiCsANJEHYgiTTX2dv99hlzzxNPPNHxuocOHaqwk7mBBABJEHYgCcIOJEHYgSQIO5AEYQeSIOxAEvPmOvvKlStL60uWLOlRJ+iVCy+8sON1d+zYUWEnc0Pbkd32U7ZP2t47bdli2ztsHyruF9XbJoBuzeYwfrOkdWcse0DSzohYIWln8RxAH2sb9oh4WdJ7ZyzeIGlL8XiLpNsq7gtAxTo9Z18SEROSFBETti9t9ULbg5IGO9wOgIrU/gFdRAxJGpIk21H39gDMrNNLbydsD0hScX+yupYA1KHTsG+XtKl4vEnS89W0A6AubQ/jbT8jabWki22PS/qhpEck/cr23ZKOSrq9ziZnY/369aX18847r0edoCrtvhtRNv96O8eOHet43bmqbdgjYmOL0pqKewFQI74uCyRB2IEkCDuQBGEHkiDsQBLz5ieuV111VVfr79u3r6JOUJXHHnustN7u0tzBgwdb1k6dOtVRT3MZIzuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJDFvrrN3a2RkpOkW5qQLLrigtL5u3Zl/q/T/3XnnnaXrrl27tqOepjz88MMta++//35X7z0XMbIDSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBJcZy8sXry4sW1fffXVpXXbpfVbb721ZW3p0qWl6y5cuLC0fscdd5TWFywoHy8+/PDDlrVdu3aVrvvRRx+V1s89t/yf7+7du0vr2TCyA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EASjojebcyubWOPP/54af3ee+8trbf7ffPRo0fPuqfZWrlyZWm93XX206dPt6x98MEHpevu37+/tN7uWvjo6Ghp/aWXXmpZO3HiROm64+PjpfVFixaV1tt9h2C+iogZ/8G0HdltP2X7pO2905Y9ZPuY7T3FrXxydACNm81h/GZJM/25kZ9ExDXF7TfVtgWgam3DHhEvS3qvB70AqFE3H9Ddb/v14jC/5cmT7UHbo7bLT+4A1KrTsP9M0pclXSNpQtKPWr0wIoYiYlVErOpwWwAq0FHYI+JERHwcEZ9I+rmk66ttC0DVOgq77YFpT78haW+r1wLoD21/z277GUmrJV1se1zSDyWttn2NpJB0RFL5ReweuO+++0rrb7/9dmn9xhtvrLKds9LuGv5zzz1XWh8bG2tZe/XVVzvqqRcGBwdL65dccklp/fDhw1W2M++1DXtEbJxh8ZM19AKgRnxdFkiCsANJEHYgCcIOJEHYgSTS/CnpRx99tOkWcIY1a9Z0tf62bdsq6iQHRnYgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSCLNdXbMP8PDw023MKcwsgNJEHYgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EAS/J4dfct2af3KK68srffzdNVNaDuy215m+/e2x2zvs/29Yvli2ztsHyruF9XfLoBOzeYw/rSkf4uIr0j6J0nftf1VSQ9I2hkRKyTtLJ4D6FNtwx4RExHxWvH4lKQxSZdJ2iBpS/GyLZJuq6tJAN07q3N221dIulbSLklLImJCmvwPwfalLdYZlDTYXZsAujXrsNv+vKRtkr4fEX9p9+HJlIgYkjRUvEd00iSA7s3q0pvtz2ky6Fsj4tli8QnbA0V9QNLJeloEUIXZfBpvSU9KGouIH08rbZe0qXi8SdLz1beHzCKi9LZgwYLSGz5tNofxN0n6tqQ3bO8plj0o6RFJv7J9t6Sjkm6vp0UAVWgb9oj4o6RWJ+hrqm0HQF041gGSIOxAEoQdSIKwA0kQdiAJfuKKOeuGG24orW/evLk3jcwRjOxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kATX2dG3ZvvXkDA7jOxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kATX2dGYF154obR+++38dfIqMbIDSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKOiPIX2Msk/ULSFyV9ImkoIn5q+yFJ90j6n+KlD0bEb9q8V/nGAHQtImb8QwCzCfuApIGIeM32FyTtlnSbpG9K+mtEPDbbJgg7UL9WYZ/N/OwTkiaKx6dsj0m6rNr2ANTtrM7ZbV8h6VpJu4pF99t+3fZTthe1WGfQ9qjt0a46BdCVtofxf3+h/XlJL0n6j4h41vYSSe9KCkkPa/JQ/ztt3oPDeKBmHZ+zS5Ltz0n6taQXI+LHM9SvkPTriPiHNu9D2IGatQp728N4T/6JzycljU0PevHB3ZRvSNrbbZMA6jObT+O/JukPkt7Q5KU3SXpQ0kZJ12jyMP6IpHuLD/PK3ouRHahZV4fxVSHsQP06PowHMD8QdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkuj1lM3vSnp72vOLi2X9qF9769e+JHrrVJW9Xd6q0NPfs39m4/ZoRKxqrIES/dpbv/Yl0VunetUbh/FAEoQdSKLpsA81vP0y/dpbv/Yl0VunetJbo+fsAHqn6ZEdQI8QdiCJRsJue53tA7bfsv1AEz20YvuI7Tds72l6frpiDr2TtvdOW7bY9g7bh4r7GefYa6i3h2wfK/bdHtvrG+ptme3f2x6zvc/294rlje67kr56st96fs5u+xxJByV9XdK4pBFJGyNif08bacH2EUmrIqLxL2DYvlnSXyX9YmpqLdv/Kem9iHik+I9yUUT8e5/09pDOchrvmnprNc34v6rBfVfl9OedaGJkv17SWxFxOCL+JumXkjY00Effi4iXJb13xuINkrYUj7do8h9Lz7XorS9ExEREvFY8PiVpaprxRvddSV890UTYL5P0zrTn4+qv+d5D0u9s77Y92HQzM1gyNc1WcX9pw/2cqe003r10xjTjfbPvOpn+vFtNhH2mqWn66frfTRHxj5L+RdJ3i8NVzM7PJH1Zk3MATkj6UZPNFNOMb5P0/Yj4S5O9TDdDXz3Zb02EfVzSsmnPl0o63kAfM4qI48X9SUnDmjzt6CcnpmbQLe5PNtzP30XEiYj4OCI+kfRzNbjvimnGt0naGhHPFosb33cz9dWr/dZE2EckrbC93PZCSd+StL2BPj7D9vnFByeyfb6kteq/qai3S9pUPN4k6fkGe/mUfpnGu9U042p43zU+/XlE9Pwmab0mP5H/s6QfNNFDi76+JOlPxW1f071JekaTh3X/q8kjorslXSRpp6RDxf3iPurtvzQ5tffrmgzWQEO9fU2Tp4avS9pT3NY3ve9K+urJfuPrskASfIMOSIKwA0kQdiAJwg4kQdiBJAg7kARhB5L4P2DL5W//DUIbAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAMdUlEQVR4nO3dX6gc5R3G8eeJbRGiaDQYo02NFS9aio0lSMFQUkxDFCHxomIuSqTS40WVChUiVlAphVBri4gKp2j+lNZSiDahlKqEqC1B8SipxiapNkRNcjinIqK5SvX8enEmcoy7s8edmZ1Nft8PHHZ33t2ZH0OevO/M7M7riBCAU9+ctgsAMBiEHUiCsANJEHYgCcIOJPGFQW7MNqf+gYZFhDstr9Sz215le7/tN23fUWVdAJrlfq+z2z5N0r8lfU/SIUkvSVobEf8q+Qw9O9CwJnr2KyS9GREHIuKYpD9KWl1hfQAaVCXsF0p6Z8brQ8WyT7E9YnvM9liFbQGoqMoJuk5Dhc8M0yNiVNKoxDAeaFOVnv2QpEUzXn9Z0pFq5QBoSpWwvyTpUtsX2/6SpBskba+nLAB163sYHxEf2b5F0lOSTpP0WES8XltlAGrV96W3vjbGMTvQuEa+VAPg5EHYgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJEHYgSQIO5AEYQeSIOxAEn1P2Qw07a677iptv/fee0vb58zp3pctX7689LPPPfdcafvJqFLYbR+U9KGkjyV9FBFL6ygKQP3q6Nm/GxHv1rAeAA3imB1IomrYQ9LTtl+2PdLpDbZHbI/ZHqu4LQAVVB3GXxkRR2yfJ+kZ2/si4vmZb4iIUUmjkmQ7Km4PQJ8q9ewRcaR4nJT0pKQr6igKQP36DrvtubbPPP5c0kpJe+oqDEC9qgzjF0h60vbx9fwhIv5WS1VI4cYbbyxtX79+fWn71NRU39uOyHdE2XfYI+KApG/WWAuABnHpDUiCsANJEHYgCcIOJEHYgST4iStac9FFF5W2n3766QOqJAd6diAJwg4kQdiBJAg7kARhB5Ig7EAShB1IguvsaNSKFSu6tt16662V1r1v377S9muvvbZr28TERKVtn4zo2YEkCDuQBGEHkiDsQBKEHUiCsANJEHYgCa6zo5Jly5aVtm/cuLFr21lnnVVp2/fdd19p+1tvvVVp/acaenYgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSILr7Khk3bp1pe0XXHBB3+t+9tlnS9u3bNnS97oz6tmz237M9qTtPTOWnWP7GdtvFI/zmi0TQFWzGcZvkrTqhGV3SNoREZdK2lG8BjDEeoY9Ip6X9N4Ji1dL2lw83yxpTc11AahZv8fsCyJiXJIiYtz2ed3eaHtE0kif2wFQk8ZP0EXEqKRRSbIdTW8PQGf9XnqbsL1QkorHyfpKAtCEfsO+XdLxay7rJG2rpxwATXFE+cja9uOSlkuaL2lC0t2S/izpT5K+IultSd+PiBNP4nVaF8P4k8z8+fNL23vdf31qaqpr2/vvv1/62euvv760fefOnaXtWUWEOy3vecweEWu7NF1VqSIAA8XXZYEkCDuQBGEHkiDsQBKEHUiCn7gmt3jx4tL2rVu3NrbtBx98sLSdS2v1omcHkiDsQBKEHUiCsANJEHYgCcIOJEHYgSS4zp7cqlUn3kv00y677LJK69+xY0fXtgceeKDSuvH50LMDSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBI9byVd68a4lfTArVlTPg3fpk2bStvnzp1b2r5r167S9rLbQfe6DTX60+1W0vTsQBKEHUiCsANJEHYgCcIOJEHYgSQIO5AEv2c/BZTd+73J+75L0oEDB0rbuZY+PHr27LYfsz1pe8+MZffYPmx7d/F3TbNlAqhqNsP4TZI63c7kNxGxpPj7a71lAahbz7BHxPOS3htALQAaVOUE3S22Xy2G+fO6vcn2iO0x22MVtgWgon7D/oikSyQtkTQu6f5ub4yI0YhYGhFL+9wWgBr0FfaImIiIjyNiStJvJV1Rb1kA6tZX2G0vnPHyOkl7ur0XwHDoeZ3d9uOSlkuab/uQpLslLbe9RFJIOijp5gZrRA/r16/v2jY1NdXotjds2NDo+lGfnmGPiLUdFj/aQC0AGsTXZYEkCDuQBGEHkiDsQBKEHUiCn7ieBJYsWVLavnLlysa2vW3bttL2/fv3N7Zt1IueHUiCsANJEHYgCcIOJEHYgSQIO5AEYQeSYMrmk8Dk5GRp+7x5Xe8K1tMLL7xQ2n711VeXth89erTvbaMZTNkMJEfYgSQIO5AEYQeSIOxAEoQdSIKwA0nwe/aTwLnnnlvaXuV20Q8//HBpO9fRTx307EAShB1IgrADSRB2IAnCDiRB2IEkCDuQBNfZh8DGjRtL2+fMae7/5F27djW2bgyXnv+KbC+yvdP2Xtuv2/5Jsfwc28/YfqN47P8OCgAaN5su4yNJP42Ir0n6tqQf2/66pDsk7YiISyXtKF4DGFI9wx4R4xHxSvH8Q0l7JV0oabWkzcXbNkta01SRAKr7XMfsthdLulzSi5IWRMS4NP0fgu3zunxmRNJItTIBVDXrsNs+Q9JWSbdFxAd2x3vafUZEjEoaLdbBDSeBlszqNK/tL2o66L+PiCeKxRO2FxbtCyWV3wIVQKt69uye7sIflbQ3In49o2m7pHWSNhSP5XP7JtZryuUVK1aUtvf6CeuxY8e6tj300EOln52YmChtx6ljNsP4KyX9QNJrtncXy+7UdMj/ZPsmSW9L+n4zJQKoQ8+wR8Q/JHU7QL+q3nIANIWvywJJEHYgCcIOJEHYgSQIO5AEP3EdgLPPPru0/fzzz6+0/sOHD3dtu/322yutG6cOenYgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1Igt+zD8C+fftK23tNm7xs2bI6y0FS9OxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kIQjovwN9iJJWySdL2lK0mhEPGD7Hkk/kvTf4q13RsRfe6yrfGMAKouIjrMuzybsCyUtjIhXbJ8p6WVJayRdL+loRPxqtkUQdqB53cI+m/nZxyWNF88/tL1X0oX1lgegaZ/rmN32YkmXS3qxWHSL7VdtP2Z7XpfPjNgesz1WqVIAlfQcxn/yRvsMSc9J+kVEPGF7gaR3JYWkn2t6qP/DHutgGA80rO9jdkmy/UVJf5H0VET8ukP7Ykl/iYhv9FgPYQca1i3sPYfxti3pUUl7Zwa9OHF33HWS9lQtEkBzZnM2fpmkv0t6TdOX3iTpTklrJS3R9DD+oKSbi5N5ZeuiZwcaVmkYXxfCDjSv72E8gFMDYQeSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgrADSRB2IIlBT9n8rqS3ZryeXywbRsNa27DWJVFbv+qs7aJuDQP9PftnNm6PRcTS1gooMay1DWtdErX1a1C1MYwHkiDsQBJth3205e2XGdbahrUuidr6NZDaWj1mBzA4bffsAAaEsANJtBJ226ts77f9pu072qihG9sHbb9me3fb89MVc+hN2t4zY9k5tp+x/Ubx2HGOvZZqu8f24WLf7bZ9TUu1LbK90/Ze26/b/kmxvNV9V1LXQPbbwI/ZbZ8m6d+SvifpkKSXJK2NiH8NtJAubB+UtDQiWv8Chu3vSDoqacvxqbVs/1LSexGxofiPcl5ErB+S2u7R55zGu6Hauk0zfqNa3Hd1Tn/ejzZ69iskvRkRByLimKQ/SlrdQh1DLyKel/TeCYtXS9pcPN+s6X8sA9eltqEQEeMR8Urx/ENJx6cZb3XfldQ1EG2E/UJJ78x4fUjDNd97SHra9su2R9oupoMFx6fZKh7Pa7meE/WcxnuQTphmfGj2XT/Tn1fVRtg7TU0zTNf/royIb0m6WtKPi+EqZucRSZdoeg7AcUn3t1lMMc34Vkm3RcQHbdYyU4e6BrLf2gj7IUmLZrz+sqQjLdTRUUQcKR4nJT2p6cOOYTJxfAbd4nGy5Xo+ERETEfFxRExJ+q1a3HfFNONbJf0+Ip4oFre+7zrVNaj91kbYX5J0qe2LbX9J0g2StrdQx2fYnlucOJHtuZJWavimot4uaV3xfJ2kbS3W8inDMo13t2nG1fK+a33684gY+J+kazR9Rv4/kn7WRg1d6vqqpH8Wf6+3XZukxzU9rPufpkdEN0k6V9IOSW8Uj+cMUW2/0/TU3q9qOlgLW6ptmaYPDV+VtLv4u6btfVdS10D2G1+XBZLgG3RAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kMT/AR1U3JCzCjB2AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAANnElEQVR4nO3dX6xV9ZnG8edRW/9RIwzgMBanBbkYNcaOBCcpESe16HghVNMREieIzdCYatqkJhrGWBM1aSbTNt7YBNBAR0aDAQc0zVhCqsgN8WgYRbFFCdPSQ8CGGCzRMMI7F2cxOcWzf+uw/60N7/eTnOx91rvXXm/24WGtvX97rZ8jQgDOfGc13QCA/iDsQBKEHUiCsANJEHYgiXP6uTHbfPQP9FhEeKzlHe3Zbd9s+ze237f9YCfPBaC33O44u+2zJf1W0jcl7ZP0uqTFEfFuYR327ECP9WLPPkfS+xGxJyKOSnpO0oIOng9AD3US9ksl/X7U7/uqZX/G9jLbQ7aHOtgWgA518gHdWIcKnztMj4gVklZIHMYDTepkz75P0vRRv39Z0nBn7QDolU7C/rqkWba/avuLkhZJ2tSdtgB0W9uH8RHxme17Jb0s6WxJT0fEO13rDEBXtT301tbGeM8O9FxPvlQD4PRB2IEkCDuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUii7fnZJcn2XkkfSzom6bOImN2NpgB0X0dhr/x9RPyxC88DoIc4jAeS6DTsIelXtt+wvWysB9heZnvI9lCH2wLQAUdE+yvbfxURw7anStos6b6I2Fp4fPsbAzAuEeGxlne0Z4+I4er2oKQXJM3p5PkA9E7bYbd9oe0vnbgvab6knd1qDEB3dfJp/CWSXrB94nn+IyL+qytdAei6jt6zn/LGeM8O9FxP3rMDOH0QdiAJwg4kQdiBJAg7kEQ3ToTBALvuuuuK9TvvvLNYnzdvXrF+5ZVXnnJPJ9x///3F+vDwcLE+d+7cYv2ZZ55pWdu+fXtx3TMRe3YgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSIKz3s4Ad9xxR8vaE088UVx38uTJxXp1CnNLr7zySrE+ZcqUlrUrrriiuG6dut6ef/75lrVFixZ1tO1BxllvQHKEHUiCsANJEHYgCcIOJEHYgSQIO5AE57MPgHPOKf8ZZs8uT467cuXKlrULLriguO7WrS0n8JEkPfroo8X6tm3bivVzzz23ZW3dunXFdefPn1+s1xkaYsax0dizA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EASjLMPgLprt69atart5968eXOxXjoXXpIOHz7c9rbrnr/TcfR9+/YV62vWrOno+c80tXt220/bPmh756hlk2xvtr27up3Y2zYBdGo8h/GrJd180rIHJW2JiFmStlS/AxhgtWGPiK2SDp20eIGkE8dIayQt7HJfALqs3ffsl0TEfkmKiP22p7Z6oO1lkpa1uR0AXdLzD+giYoWkFRIXnASa1O7Q2wHb0ySpuj3YvZYA9EK7Yd8kaUl1f4mkjd1pB0Cv1F433vazkm6QNFnSAUk/kvSfktZJukzS7yR9OyJO/hBvrOdKeRhfd0748uXLi/W6v9GTTz7ZsvbQQw8V1+10HL3Orl27WtZmzZrV0XPffvvtxfrGjTn3Qa2uG1/7nj0iFrcofaOjjgD0FV+XBZIg7EAShB1IgrADSRB2IAlOce2Chx9+uFivG1o7evRosf7yyy8X6w888EDL2ieffFJct855551XrNedpnrZZZe1rNVNufzYY48V61mH1trFnh1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkqg9xbWrGzuNT3G9+OKLW9bee++94rqTJ08u1l966aVifeHC3l3i7/LLLy/W165dW6xfe+21bW97/fr1xfrdd99drB85cqTtbZ/JWp3iyp4dSIKwA0kQdiAJwg4kQdiBJAg7kARhB5JgnH2cpk5tOcOVhoeHO3ruGTNmFOuffvppsb506dKWtVtvvbW47lVXXVWsT5gwoViv+/dTqt92223FdV988cViHWNjnB1IjrADSRB2IAnCDiRB2IEkCDuQBGEHkmCcfZxK57OXpiWWpClTphTrdddP7+XfqO47AnW9TZs2rVj/8MMP214X7Wl7nN3207YP2t45atkjtv9ge0f1c0s3mwXQfeM5jF8t6eYxlv8sIq6pfn7Z3bYAdFtt2CNiq6RDfegFQA918gHdvbbfqg7zJ7Z6kO1ltodsD3WwLQAdajfsP5c0U9I1kvZL+kmrB0bEioiYHRGz29wWgC5oK+wRcSAijkXEcUkrJc3pblsAuq2tsNsePWbyLUk7Wz0WwGConZ/d9rOSbpA02fY+ST+SdIPtaySFpL2SvtvDHgfCRx991LJWd133uuvCT5o0qVj/4IMPivXSPOWrV68urnvoUPmz1+eee65Yrxsrr1sf/VMb9ohYPMbip3rQC4Ae4uuyQBKEHUiCsANJEHYgCcIOJFH7aTzqbd++vVivO8W1Sddff32xPm/evGL9+PHjxfqePXtOuSf0Bnt2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCcfbkzj///GK9bhy97jLXnOI6ONizA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EASTNmMomPHjhXrdf9+SpeaLk3njPa1PWUzgDMDYQeSIOxAEoQdSIKwA0kQdiAJwg4kwfnsyd10001Nt4A+qd2z255u+9e2d9l+x/b3q+WTbG+2vbu6ndj7dgG0azyH8Z9J+mFE/I2kv5P0PdtXSHpQ0paImCVpS/U7gAFVG/aI2B8Rb1b3P5a0S9KlkhZIWlM9bI2khb1qEkDnTuk9u+2vSPqapO2SLomI/dLIfwi2p7ZYZ5mkZZ21CaBT4w677QmS1kv6QUQctsf8rv3nRMQKSSuq5+BEGKAh4xp6s/0FjQR9bURsqBYfsD2tqk+TdLA3LQLohto9u0d24U9J2hURPx1V2iRpiaQfV7cbe9IhemrGjBlNt4A+Gc9h/Ncl/ZOkt23vqJYt10jI19n+jqTfSfp2b1oE0A21YY+IbZJavUH/RnfbAdArfF0WSIKwA0kQdiAJwg4kQdiBJDjFNbnXXnutWD/rrPL+oG5KZwwO9uxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kATj7Mnt3LmzWN+9e3exXnc+/MyZM1vWmLK5v9izA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EASjujfJC3MCHP6ueuuu4r1VatWFeuvvvpqy9p9991XXPfdd98t1jG2iBjzatDs2YEkCDuQBGEHkiDsQBKEHUiCsANJEHYgidpxdtvTJf1C0l9KOi5pRUQ8YfsRSf8s6cRJycsj4pc1z8U4+2nmoosuKtbXrVtXrN94440taxs2bCiuu3Tp0mL9yJEjxXpWrcbZx3Pxis8k/TAi3rT9JUlv2N5c1X4WEf/WrSYB9M545mffL2l/df9j27skXdrrxgB01ym9Z7f9FUlfk7S9WnSv7bdsP217Yot1ltkesj3UUacAOjLusNueIGm9pB9ExGFJP5c0U9I1Gtnz/2Ss9SJiRUTMjojZXegXQJvGFXbbX9BI0NdGxAZJiogDEXEsIo5LWilpTu/aBNCp2rDbtqSnJO2KiJ+OWj5t1MO+Jal8mVIAjRrP0NtcSa9JelsjQ2+StFzSYo0cwoekvZK+W32YV3ouht7OMHVDc48//njL2j333FNc9+qrry7WOQV2bG0PvUXENkljrVwcUwcwWPgGHZAEYQeSIOxAEoQdSIKwA0kQdiAJLiUNnGG4lDSQHGEHkiDsQBKEHUiCsANJEHYgCcIOJDGeq8t20x8l/c+o3ydXywbRoPY2qH1J9Naubvb2160Kff1Szec2bg8N6rXpBrW3Qe1Lord29as3DuOBJAg7kETTYV/R8PZLBrW3Qe1Lord29aW3Rt+zA+ifpvfsAPqEsANJNBJ22zfb/o3t920/2EQPrdjea/tt2zuanp+umkPvoO2do5ZNsr3Z9u7qdsw59hrq7RHbf6heux22b2mot+m2f217l+13bH+/Wt7oa1foqy+vW9/fs9s+W9JvJX1T0j5Jr0taHBEDccV/23slzY6Ixr+AYft6SX+S9IuIuKpa9q+SDkXEj6v/KCdGxAMD0tsjkv7U9DTe1WxF00ZPMy5poaS71OBrV+jrH9WH162JPfscSe9HxJ6IOCrpOUkLGuhj4EXEVkmHTlq8QNKa6v4ajfxj6bsWvQ2EiNgfEW9W9z+WdGKa8UZfu0JffdFE2C+V9PtRv+/TYM33HpJ+ZfsN28uabmYMl5yYZqu6ndpwPyernca7n06aZnxgXrt2pj/vVBNhH+v6WIM0/vf1iPhbSf8g6XvV4SrGZ1zTePfLGNOMD4R2pz/vVBNh3ydp+qjfvyxpuIE+xhQRw9XtQUkvaPCmoj5wYgbd6vZgw/38v0GaxnusacY1AK9dk9OfNxH21yXNsv1V21+UtEjSpgb6+BzbF1YfnMj2hZLma/Cmot4kaUl1f4mkjQ328mcGZRrvVtOMq+HXrvHpzyOi7z+SbtHIJ/IfSPqXJnpo0dcMSf9d/bzTdG+SntXIYd3/auSI6DuS/kLSFkm7q9tJA9Tbv2tkau+3NBKsaQ31Nlcjbw3fkrSj+rml6deu0FdfXje+LgskwTfogCQIO5AEYQeSIOxAEoQdSIKwA0kQdiCJ/wOB61FSDzgKbQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "for i in range(5):\n",
    "    plt.imshow(imgs[i],cmap = 'gray')\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\86188\\appdata\\local\\programs\\python\\python38\\lib\\site-packages\\sklearn\\feature_extraction\\text.py:17: DeprecationWarning: Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated since Python 3.3, and in 3.9 it will stop working\n",
      "  from collections import Mapping, defaultdict\n"
     ]
    }
   ],
   "source": [
    "from sklearn.datasets import fetch_mldata\n",
    "mnist = fetch_mldata('MNIST original', data_home='./')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'DESCR': 'mldata.org dataset: mnist-original',\n",
       " 'COL_NAMES': ['label', 'data'],\n",
       " 'target': array([0., 0., 0., ..., 9., 9., 9.]),\n",
       " 'data': array([[0, 0, 0, ..., 0, 0, 0],\n",
       "        [0, 0, 0, ..., 0, 0, 0],\n",
       "        [0, 0, 0, ..., 0, 0, 0],\n",
       "        ...,\n",
       "        [0, 0, 0, ..., 0, 0, 0],\n",
       "        [0, 0, 0, ..., 0, 0, 0],\n",
       "        [0, 0, 0, ..., 0, 0, 0]], dtype=uint8)}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mnist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "X,y = mnist['data'],mnist['target']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(70000, 784)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(70000,)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "some_digit = X[36000]\n",
    "some_digit_image = some_digit.reshape(28,28)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAANpElEQVR4nO3db6xU9Z3H8c9XLQ+UJoB3NCAutzZg1piUkgnZxE1jbbZRicE+qMIDZJMmtw/EQMSkpE2shiekrjY1MU3oQnpduzaYlgUj2a3BJoQHVkcDgiVFivyrN9wBEnv7gHSx3z64x+YCc37nMufMnIHv+5VMZuZ858z5Zrgfzsz5zZmfubsAXPuuq7sBAP1B2IEgCDsQBGEHgiDsQBA39HNjQ0NDPjw83M9NAqEcO3ZMZ86csU61UmE3s/sl/UTS9ZL+0903pR4/PDysVqtVZpMAEprNZm6t67fxZna9pJckPSDpLkkrzeyubp8PQG+V+cy+VNIRdz/q7n+V9EtJy6tpC0DVyoT9Nkknp9w/lS27iJmNmFnLzFrtdrvE5gCUUSbsnQ4CXPbdW3ff7O5Nd282Go0SmwNQRpmwn5J0+5T78yV9Uq4dAL1SJuzvSlpoZl8ysxmSVkjaWU1bAKrW9dCbu18wszWS/k+TQ29b3f3DyjoDUKlS4+zuvkvSrop6AdBDfF0WCIKwA0EQdiAIwg4EQdiBIAg7EARhB4Ig7EAQhB0IgrADQRB2IAjCDgRB2IEgCDsQBGEHgiDsQBCEHQiCsANBEHYgCMIOBEHYgSAIOxAEYQeCIOxAEIQdCIKwA0EQdiAIwg4EQdiBIErN4grUadu2bcn6gQMHcmsvv/xy1e1c5Pjx4z19/m6UCruZHZM0IekzSRfcvVlFUwCqV8We/evufqaC5wHQQ3xmB4IoG3aX9Bsze8/MRjo9wMxGzKxlZq12u11ycwC6VTbs97j7EkkPSHrczL526QPcfbO7N9292Wg0Sm4OQLdKhd3dP8muxyVtl7S0iqYAVK/rsJvZTWb2xc9vS/qmpINVNQagWmWOxt8qabuZff48/+3u/1tJV7hmTExM5Nb27t2bXHfjxo3J+ttvv52sZ3+byHQddnc/KukrFfYCoIcYegOCIOxAEIQdCIKwA0EQdiAITnG9xl24cCFZHxsbK/X8RcNjH3/8cW7trbfeKrXtXhoaGkrWV6xY0adOqsOeHQiCsANBEHYgCMIOBEHYgSAIOxAEYQeCYJz9Glc0jj48PJysu3uyPsinkS5evDi3tmrVquS6y5YtS9YXLlzYVU91Ys8OBEHYgSAIOxAEYQeCIOxAEIQdCIKwA0Ewzn6Ne+qpp5L1onH0onqRefPm5dZGRjrOGPYPTz/9dKlt42Ls2YEgCDsQBGEHgiDsQBCEHQiCsANBEHYgCMbZrwFbt27Nre3atSu5btnz0YvWP3v2bG6t6DftDx8+nKwvWrQoWcfFCvfsZrbVzMbN7OCUZXPM7E0z+yi7nt3bNgGUNZ238T+XdP8lyzZI2u3uCyXtzu4DGGCFYXf3PZLOXbJ4uaTR7PaopIcr7gtAxbo9QHeru49JUnZ9S94DzWzEzFpm1mq3211uDkBZPT8a7+6b3b3p7s1Go9HrzQHI0W3YT5vZXEnKrserawlAL3Qb9p2SVme3V0vaUU07AHrFpvG74K9KulfSkKTTkn4o6X8kbZP0T5JOSPq2u196EO8yzWbTW61WyZbjSY2jS9KTTz6ZW5uYmCi17Tp/N37BggXJ+tGjR3u27atVs9lUq9Xq+I9S+KUad1+ZU/pGqa4A9BVflwWCIOxAEIQdCIKwA0EQdiAITnG9Cjz77LPJepnhtVmzZiXrM2fOTNavuy69vzh//nxubXw8/V2s48ePJ+u4MuzZgSAIOxAEYQeCIOxAEIQdCIKwA0EQdiAIxtmvAsuXL0/WX3rppdza6tWrc2uStGbNmmR9yZIlyXqRsbGx3NqyZcuS6+7fv7/UtnEx9uxAEIQdCIKwA0EQdiAIwg4EQdiBIAg7EATj7FeBF198sVS9Tqmfoi76meqiOq4Me3YgCMIOBEHYgSAIOxAEYQeCIOxAEIQdCIJx9szJkyeT9RtvvDG3dvPNN1fdzjUjdU560XTPRfUdO3Yk60W/AxBN4Z7dzLaa2biZHZyy7Bkz+5OZ7csuD/a2TQBlTedt/M8l3d9h+Y/dfXF22VVtWwCqVhh2d98j6VwfegHQQ2UO0K0xsw+yt/mz8x5kZiNm1jKzVrvdLrE5AGV0G/afSvqypMWSxiQ9n/dAd9/s7k13bzYajS43B6CsrsLu7qfd/TN3/5ukn0laWm1bAKrWVdjNbO6Uu9+SdDDvsQAGQ+E4u5m9KuleSUNmdkrSDyXda2aLJbmkY5K+28MeK7Fp06ZkfXR0NFmfMWNGbu2OO+5Irrt9+/Zk/Wp29uzZZH3Dhg25tYMH0/uI4eHhblpCjsKwu/vKDou39KAXAD3E12WBIAg7EARhB4Ig7EAQhB0IIswpru+8806yfvjw4a6f+8SJE8n6+vXrk/Xnn8/9AmLtik79feONN5L11PDaDTek//zuvvvuZJ1TWK8Me3YgCMIOBEHYgSAIOxAEYQeCIOxAEIQdCCLMOHsvzZo1K1kf5HH0ImvXrk3Wi37OOWXevHk9e25cjj07EARhB4Ig7EAQhB0IgrADQRB2IAjCDgQRZpy96GeJZ86cmaxPTEzk1h566KFuWuqLRx99NFl/7bXXknV3T9aLplVOee6557peF1eOPTsQBGEHgiDsQBCEHQiCsANBEHYgCMIOBBFmnP2FF15I1o8cOZKsp34f/fz588l1i8ayi2zcuDFZ//TTT3Nr586dS65bNE5+5513JuuPPfZY1/U5c+Yk10W1CvfsZna7mf3WzA6Z2YdmtjZbPsfM3jSzj7Lr2b1vF0C3pvM2/oKk9e7+z5L+RdLjZnaXpA2Sdrv7Qkm7s/sABlRh2N19zN3fz25PSDok6TZJyyWNZg8blfRwr5oEUN4VHaAzs2FJX5X0O0m3uvuYNPkfgqRbctYZMbOWmbXa7Xa5bgF0bdphN7OZkn4laZ27/3m667n7Zndvunuz0Wh00yOACkwr7Gb2BU0G/Rfu/uts8Wkzm5vV50oa702LAKpQOPRmk2MzWyQdcvep41c7Ja2WtCm7vqp/93fdunXJempa5t27dyfX3bJlS7Ley9NIFy1alKwPDQ0l66+88kqyvmDBgivuCfWYzjj7PZJWSTpgZvuyZd/XZMi3mdl3JJ2Q9O3etAigCoVhd/e9kvJ2Ld+oth0AvcLXZYEgCDsQBGEHgiDsQBCEHQgizCmuRe67775kPTWWXnQa6f79+5P1PXv2JOuvv/56sv7EE0/k1h555JHkuvPnz0/Wce1gzw4EQdiBIAg7EARhB4Ig7EAQhB0IgrADQVjRudRVajab3mq1+rY9IJpms6lWq9XxLFX27EAQhB0IgrADQRB2IAjCDgRB2IEgCDsQBGEHgiDsQBCEHQiCsANBEHYgCMIOBEHYgSAIOxBEYdjN7HYz+62ZHTKzD81sbbb8GTP7k5ntyy4P9r5dAN2aziQRFyStd/f3zeyLkt4zszez2o/d/T961x6AqkxnfvYxSWPZ7QkzOyTptl43BqBaV/SZ3cyGJX1V0u+yRWvM7AMz22pms3PWGTGzlpm12u12qWYBdG/aYTezmZJ+JWmdu/9Z0k8lfVnSYk3u+Z/vtJ67b3b3prs3G41GBS0D6Ma0wm5mX9Bk0H/h7r+WJHc/7e6fufvfJP1M0tLetQmgrOkcjTdJWyQdcvcXpiyfO+Vh35J0sPr2AFRlOkfj75G0StIBM9uXLfu+pJVmtliSSzom6bs96RBAJaZzNH6vpE6/Q72r+nYA9ArfoAOCIOxAEIQdCIKwA0EQdiAIwg4EQdiBIAg7EARhB4Ig7EAQhB0IgrADQRB2IAjCDgRh7t6/jZm1JR2fsmhI0pm+NXBlBrW3Qe1LorduVdnbAnfv+PtvfQ37ZRs3a7l7s7YGEga1t0HtS6K3bvWrN97GA0EQdiCIusO+uebtpwxqb4Pal0Rv3epLb7V+ZgfQP3Xv2QH0CWEHgqgl7GZ2v5n9wcyOmNmGOnrIY2bHzOxANg11q+ZetprZuJkdnLJsjpm9aWYfZdcd59irqbeBmMY7Mc14ra9d3dOf9/0zu5ldL+mwpH+TdErSu5JWuvvv+9pIDjM7Jqnp7rV/AcPMvibpL5Jedve7s2U/knTO3Tdl/1HOdvfvDUhvz0j6S93TeGezFc2dOs24pIcl/btqfO0SfT2iPrxudezZl0o64u5H3f2vkn4paXkNfQw8d98j6dwli5dLGs1uj2ryj6XvcnobCO4+5u7vZ7cnJH0+zXitr12ir76oI+y3STo55f4pDdZ87y7pN2b2npmN1N1MB7e6+5g0+ccj6Zaa+7lU4TTe/XTJNOMD89p1M/15WXWEvdNUUoM0/nePuy+R9ICkx7O3q5ieaU3j3S8dphkfCN1Of15WHWE/Jen2KffnS/qkhj46cvdPsutxSds1eFNRn/58Bt3serzmfv5hkKbx7jTNuAbgtatz+vM6wv6upIVm9iUzmyFphaSdNfRxGTO7KTtwIjO7SdI3NXhTUe+UtDq7vVrSjhp7ucigTOOdN824an7tap/+3N37fpH0oCaPyP9R0g/q6CGnrzsk7c8uH9bdm6RXNfm27v81+Y7oO5JulrRb0kfZ9ZwB6u2/JB2Q9IEmgzW3pt7+VZMfDT+QtC+7PFj3a5foqy+vG1+XBYLgG3RAEIQdCIKwA0EQdiAIwg4EQdiBIAg7EMTfAa5yOtysgto/AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(some_digit_image,cmap = matplotlib.cm.binary)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5.0"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y[36000]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 建立测试集和训练集"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train,X_test,y_train,y_test = X[:60000],X[60000:],y[:60000],y[60000:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([11839,  2018, 48897, ..., 35397, 31089, 46971])"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "shuffle_index = np.random.permutation(60000)\n",
    "shuffle_index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train,y_train = X[shuffle_index],y[shuffle_index]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([False, False, False, ...,  True,  True, False])"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 识别数字5，二元分类5或非5\n",
    "y_train_5 = (y_train == 5)\n",
    "y_train_5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[False, False, False, ..., False, False, False],\n",
       "       [False, False, False, ...,  True, False, False],\n",
       "       [False,  True, False, ..., False, False, False],\n",
       "       ...,\n",
       "       [False,  True, False, ..., False, False, False],\n",
       "       [False, False, False, ..., False, False, False],\n",
       "       [False, False, False, ...,  True,  True, False]])"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train_5.reshape(20,-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([False, False, False, ..., False, False, False])"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_test_5 = (y_test == 5)\n",
    "y_test_5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\86188\\appdata\\local\\programs\\python\\python38\\lib\\site-packages\\sklearn\\linear_model\\stochastic_gradient.py:123: FutureWarning: max_iter and tol parameters have been added in <class 'sklearn.linear_model.stochastic_gradient.SGDClassifier'> in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([ True])"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# SGD梯度下降分类器，适合非常大的数据集，独立处理训练集数据，适合在线学习\n",
    "from sklearn.linear_model import SGDClassifier\n",
    "sgd_clf = SGDClassifier(random_state = 42)\n",
    "sgd_clf.fit(X_train,y_train_5)\n",
    "\n",
    "sgd_clf.predict([some_digit])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 性能考核"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 使用交叉验证测量精度"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\86188\\appdata\\local\\programs\\python\\python38\\lib\\site-packages\\sklearn\\linear_model\\stochastic_gradient.py:123: FutureWarning: max_iter and tol parameters have been added in <class 'sklearn.linear_model.stochastic_gradient.SGDClassifier'> in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n",
      "  warnings.warn(\n",
      "c:\\users\\86188\\appdata\\local\\programs\\python\\python38\\lib\\site-packages\\sklearn\\linear_model\\stochastic_gradient.py:123: FutureWarning: max_iter and tol parameters have been added in <class 'sklearn.linear_model.stochastic_gradient.SGDClassifier'> in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n",
      "  warnings.warn(\n",
      "c:\\users\\86188\\appdata\\local\\programs\\python\\python38\\lib\\site-packages\\sklearn\\linear_model\\stochastic_gradient.py:123: FutureWarning: max_iter and tol parameters have been added in <class 'sklearn.linear_model.stochastic_gradient.SGDClassifier'> in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([0.95145, 0.9639 , 0.9608 ])"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## 评估分类器比评估回归器要困难的多\n",
    "# 三个折叠，正确率达到95%以上\n",
    "from sklearn.model_selection import cross_val_score\n",
    "cross_val_score(sgd_clf,X_train,y_train_5,cv=3,scoring = 'accuracy')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 把每张图都分类为非5\n",
    "from sklearn.base import BaseEstimator\n",
    "class Never5Classifier(BaseEstimator):\n",
    "    def fit(self,X,y = None):\n",
    "        pass\n",
    "    def predict(self,X):\n",
    "        return np.zeros((len(X),1),dtype =bool)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.912  , 0.9064 , 0.91055])"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "never_5_clf =  Never5Classifier()\n",
    "cross_val_score(never_5_clf,X_train,y_train_5,cv=3,scoring = 'accuracy')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 混淆矩阵"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\86188\\appdata\\local\\programs\\python\\python38\\lib\\site-packages\\sklearn\\linear_model\\stochastic_gradient.py:123: FutureWarning: max_iter and tol parameters have been added in <class 'sklearn.linear_model.stochastic_gradient.SGDClassifier'> in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n",
      "  warnings.warn(\n",
      "c:\\users\\86188\\appdata\\local\\programs\\python\\python38\\lib\\site-packages\\sklearn\\linear_model\\stochastic_gradient.py:123: FutureWarning: max_iter and tol parameters have been added in <class 'sklearn.linear_model.stochastic_gradient.SGDClassifier'> in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n",
      "  warnings.warn(\n",
      "c:\\users\\86188\\appdata\\local\\programs\\python\\python38\\lib\\site-packages\\sklearn\\linear_model\\stochastic_gradient.py:123: FutureWarning: max_iter and tol parameters have been added in <class 'sklearn.linear_model.stochastic_gradient.SGDClassifier'> in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import cross_val_predict\n",
    "y_train_pred = cross_val_predict(sgd_clf,X_train,y_train_5,cv=3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 与cross_val_score相比\n",
    "   * 同样执行交叉验证\n",
    "   * 返回的不是评估分数，而是每个折叠的预测\n",
    "   * 每一个实例在模型与测试使用的数据，在训练期间从未见过"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[53336,  1243],\n",
       "       [ 1234,  4187]], dtype=int64)"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 获得混淆矩阵\n",
    "from sklearn.metrics import confusion_matrix\n",
    "confusion_matrix(y_train_5,y_train_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 行表示实际类别，列表示预测类别"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[54579,     0],\n",
       "       [    0,  5421]], dtype=int64)"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train_perfext_predictions = y_train_5\n",
    "confusion_matrix(y_train_5,y_train_perfext_predictions)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 精度和召回率"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7710865561694291"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import precision_score,recall_score\n",
    "\n",
    "precision_score(y_train_5,y_train_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7723667220070097"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "recall_score(y_train_5,y_train_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7717261081927934"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 合成为单一指标谐波平均值F1，谐波平均值会给予较低值更高权重\n",
    "from sklearn.metrics import f1_score\n",
    "f1_score(y_train_5,y_train_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # 如何设置阀值\n",
    "# # 用predict_proba得到每个实例属于正类的概率，然后对概率切一下，以LogisticRegression为例\n",
    "# clf = LogisticRegression()\n",
    "# clf.fit(X_train,y_train)\n",
    "# pred_proda = clf.predict_proba(X_test)[ : , 1]\n",
    "# threshold = 0.75  # 设置阀值\n",
    "# pred_label = pred_proda > threshold\n",
    "# # pred_proba是每个实例为真的概率，假设阀值为0.75，pred_label里True就是概率大于0.75的"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([20200.84266059])"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 返回决策值decision_function\n",
    "y_scores = sgd_clf.decision_function([some_digit])\n",
    "y_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "threshold = 0\n",
    "y_some_digit_pred = (y_scores > threshold)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ True])"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_some_digit_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ True])"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 提高阀值可以降低召回率，提高阀值到10000，就错过了这个图\n",
    "threshold = 10000\n",
    "y_some_digit_pred = (y_scores > threshold)\n",
    "y_some_digit_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\86188\\appdata\\local\\programs\\python\\python38\\lib\\site-packages\\sklearn\\linear_model\\stochastic_gradient.py:123: FutureWarning: max_iter and tol parameters have been added in <class 'sklearn.linear_model.stochastic_gradient.SGDClassifier'> in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n",
      "  warnings.warn(\n",
      "c:\\users\\86188\\appdata\\local\\programs\\python\\python38\\lib\\site-packages\\sklearn\\linear_model\\stochastic_gradient.py:123: FutureWarning: max_iter and tol parameters have been added in <class 'sklearn.linear_model.stochastic_gradient.SGDClassifier'> in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n",
      "  warnings.warn(\n",
      "c:\\users\\86188\\appdata\\local\\programs\\python\\python38\\lib\\site-packages\\sklearn\\linear_model\\stochastic_gradient.py:123: FutureWarning: max_iter and tol parameters have been added in <class 'sklearn.linear_model.stochastic_gradient.SGDClassifier'> in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "# 如何决定使用什么阀值是和需求决定的\n",
    "# 返回决策值，而不是预测结果\n",
    "y_scores = cross_val_predict(sgd_clf,X_train,y_train_5,cv=3,method='decision_function')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(60000,)"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_scores.shape # 有了y_scores，可以计算所有可能的阀值的精度和召回率"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import precision_recall_curve\n",
    "precisions,recalls,thresholds = precision_recall_curve(y_train_5,y_scores)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## PR曲线"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAeMAAAEPCAYAAABx8azBAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nOzdd3xO5//H8deVPWSQYcWIXWpVbEVrtmqUompVqZZqS8dX268q1X611Wp1KKHEqr1rFVW0+BGrqBWxVxYiMu/k/P64yCAIkpw7yef5eJyHc59z3ed87lS9c537OtdRhmEghBBCCPPYmF2AEEIIUdBJGAshhBAmkzAWQgghTCZhLIQQQphMwlgIIYQwmYSxEEIIYbL7hrFSappSKkwpdfAu+5VS6nulVIhS6h+l1BPZX6YQQgiRf2WlZxwEtL3H/meAijeXgcDPj16WEEIIUXDcN4wNw9gCRN2jSUdgpqHtADyVUsWzq0AhhBAiv7PLhmOUBM6me33u5raLtzdUSg1E955xcXWpU7lyZRQKpVQ2lCGEEEJYt927d0cYhuFz+/bsCOPMkjTTOTYNwwgEAgFUCWXs7bA3dV9hp8K4O7rj7eKNva09tsoWG2WDrc3NP5UtjnaOONk5YatssbWxTf3TBhvsbOxwc3TD2c4ZJzsnCjkUws7GLkO7W8dwsXe58xjKBmc7Z3xcffBw9MDOxi7D++1s7OSXBiGEyGFbt8KJE+DmBu7uULIkVK364MdJSYE33oAzZ8DDQx/PzU1v8/fXbfbtg3PnoFChtP1ublC4MDg5PXjd48aBYYCLCzg7Z/zz1VehdGlQSp3O7P3ZEcbngFLpXvsBF+73JlsbW5zsnUhMTiQpJYkr8Ve4En+F09cyrdMqONg6ZPrLgK2yxcnOCWd7Z+xt7LG3tde/HDi44WjniL2NPQ62Dng6eVLIoRAejh442TnhaOeIq70r3i7eeLt4U8ihEMUKFaOIcxEJfiGEVUtIgJMndZC5u+s/bWx0GMXF6QACOHoU4uOhSBG9/Pwz/PqrDsg5c6BECd1u7Vro3Fm/N73nnoOVK/V6eDiUKaOP4+sLPj5QtKg+RvHi0KmT3r9gAXTvnnnd3bunhfHkyTBpUubt6tSB4OC01wMG6JAuWlSfu2hRKFZMn9fbG7ZsgV274NKlzI/Xvr0O47vJjjBeAQxRSs0D6gPXDMO44xL17WoVq0XwR/qTWlIsXI2/yqWYS8QmxZKckkyykUyKkZK6npySzI2kG1hSLBm2pRgpJBvJJFgSiEmMISE5gRuJN/RxbrZJNjIeIzE5McMxbp0rOiGaqLgoohOiSU5J1udK1yYxOZHE5MRs+JHdm72NPc72zng4elC9aHVKuZfC08kTd0d3nO2ccXd0x8HWATsbO+xt7XG0daSws76y4OPig4+rD3Y22fGfVgiRV0RGgsWi18+cgX//1SFZsyaUK5ex7fnzEBKig6toUb0tLk6HXHqGARs2QOvW8Nhj8Pvv4Oen97VtC3/+mdZWKR1K4eHwv//Bhx/q7f/9LyxenHnNtrZp6xMmZAziTp3g+nWoVSttW3i4bnP+vF5uV726/kxXr6ZtK10aPv9cHysmRu+/pVo1ePZZvS/9cuVK2i8Tt342v/yS+WcAmDZNf87WreHCBd0+Njbtz9jYewcxZCGMlVJzgeaAt1LqHPAJYA9gGMYkYDXwLBACxAL97nfMO4qwsUvtHVorwzCITYolKSXpjiBPTkkmJjGGpJQkLCkWkpKTSEhO4HrCdZJSkkhKTiLOEkd0QjTXE65zLeFaarBfjb9KRGwEV+KvEJMYw7noc/pYCUlEJ0RzNvrs/Yu7jau9K6U9SlPCrQSuDq642uvFwdYBB1sHvF28KeVRinKFy1HYqTBFnIvgbO+Ms50zjnaOOfDTE0Lci8UCoaH6H3JnZ6hfX29PSNCXbJ2ddc/z1Cm4fBm8vPS2mjV1CI4aBaNHZ37sL7+E//xHr69YoUMus4f1ffaZDhSAjRv18bZuTdt/+DBUqQJ79kClStCsmQ5jX18dOtev67AEuHgRrl3Tvd9SpXRIRkbqJSFB90zHj9ef45bWrfWxqlSB4cN1L/t2jz2mAzUiAsLC9PkuX9bnu3ABypfX7QYMgB49dAj6+uqfUWaGDNHL7QxD9+ZvsbHRYRwWps93+bJev3RJL56eul3dupmfJyuUWY9QDAgIMILTXwMQgA79xORE4i3xXL5xmX2X9hEZG0lEbASxSbHEJMYQkxRDUrIOfkuKhRtJN1J79ZGxkYTHhj/0+R1tHXG2d8bF3iX1MruznTMVvSri7axDvIRbidRfngo7FcbX1Rdne+ds/CkIkbdFR+vgtLXVIWoYkJQEFSvq/TExOpCSk/USHZ323g8/hBEjdM+sb1+YOTPzcwwcqC+zpqToHu21a/qyrcWiv+90cNCBsWCBvtQLuvc5dGjaMfz8dJvERP0LwI4devvkyfD66xnPN26cvsRb6uaXkpGRad+Hgj7vrXAsVAgqVMi8bosF7ArwhTul1G7DMALu2C5hnP9ExkYSEhXC1firxFviuRp/lYTkBBKTE4lLiiPsRhihV0M5H32eqLgoriVcI94ST2xSLJYUy0Od09nOOfU77/JFyhNQPIAmpZtQ2qM0ZTzLYKNksjeRt8XGwl9/6cA5d04HiouLvuz5zDO69xUbC66umb+/TBkd0KCP4eubebvu3fVl3nLlICgI+vXT31Xa2uoeaEqK3hcerr+jLF1af3dbrJgOx3tJTtYhGhWle5H29voXhehofUn7Vg8yLExf5j5zRveCGzR4mJ+YyIyEsbivFCOFBEsC8ZZ4riVcIy4pjoTkBK7FXyMkKoSLMRe5FHOJSzGXCI8N50rcFSJiI4iIjSApJemuxy3kUIhS7qXwL+yPv6c/ZTzK4OfuR9MyTSnpXjIXP6EQ95eUBGfPwvbtOhjt7HRP1s0t8/YvvKB7nCVK6NG5tWtn3O/lpXuqXl665+nsrC+ztmkDffpAr166F3u344v85W5hbNUXC65du0ZERASJiTk/aEpkzhFHfPHF19kXnIE77o4DAwPDMFIH3CWlJBFviU+9lJ5ipNz5phg4cOgAB9VBbJS+Nc3exl7fZmZnQ9kSZSnqVTTHP58QsbG6J7pqlQ7TW9q315d+Bw1KG1jk46NH2bq66vC9elVfdl21St+64uAAgYG6J1m9+t3P6e0Nu3fn7OcSeYvVhnF8fDyXL1/Gz88PZ2dnudUnjzIMQw9qS0lK7XVbUizEWeKISYy5M6gNwAK7j+/ml3W/8ITfEzzl/xRVvKvg7uguo8TFQzt+XA/C2bcPFi3S32umpOjQDA29s/2FC7qXrJS+THzyJJQte+9zVK36cPfECmG1/7KFh4fj4+ODS/rx5SLPUUphb6vvvXaxz/jf8lZv+tYgtMTkRBIsCdxIuoGPtw81w2oyYtMI2KTb29nYUc2nGpW8KlHYqTBVfarS+bHOlPIolcmZRUG3caMe+RsSor///PvvtFHEkyfDu+/qUbINGugwdnLS95VWqqS/S03Pzu7+QSzEo7DaMI6Pj6dYsWJmlyFykFIqdeKU9LdUGYZBbKFYjFiDY5Zj7Lm4hwvXL3At4Rr7L+9n/+X9qW2HrhtKWc+yVCxSkeq+1XmyzJNU9qpM+SLlcbB1MONjCStw5gwsXQo//ZRxe6NG0K2bXm6ZOFFPPiGEmaw2jC0WC3YFefx7AaaUwsXJBTd7N2Z3np26/XrCdfZe2suF6xe4eP0iS48sZce5HZy6eopTV0+xPnQ943eMB3QvumKRitQoWoPmZZvTvGxzKntVlq878qGUFH0PbFCQvke1f389wrhCBd3DffJJPcjq2WczTvhwi4dHrpcsxB2sdjT14cOHeeyxx3KxImFtsvJ3ICYxhpNXTnI08ih7Lu7h77N/c/LKyUwnSynqWpRXar/C0AZD8XW9y30lIk84cUKH7oED+jad9N5+G777zpy6hLifPHdrk4SxeJS/A5GxkZy6eort57bz27HfCL4QTGRcJAC2ypbaxWtTp3gdSnuUpl3FdtQsVjM7Sxc5zNk54wxJvr76O92yZWHGjAef5F+I3CJhLPKc7Pw7YBgGG0I38NW2r9h0chPJRnKG/RWKVKBN+TbUKlaLzo91pohzkbscSZhh7Fh9OXnwYH0r0Xvv6Xt7u3WD6dMzziMshDW7WxjLtEi5KCgoCKVU6uLm5kbNmjX58ccfsVgebuarB3Xq1CmUUgQFBWX5PbfqPnVr+qA8SClFq/KtWN97PeHvh7Oh9wbGtx5P/9r98XD0ICQqhJ92/cSrK1/F+ytvWs5syVd/f8XlmMuY9QtrQXf5ctqTbj76CJYs0dvt7PRlaMOA+fMliEX+ICOkTLBw4UL8/PyIjo5m4cKFvPnmm4SFhfHpp5/m+LmLFy/O9u3bKX9rRvUsaNeuHdu3b6d48eI5WFnuKexcmBblWtCiXAsAfnr2J9aHrmffpX2sCVnDtrPb2HhyIxtPbmT4huH4ufvRr1Y/Wvi3oL5ffZzs5BpoTjIMHbbvvJNx+6VLep+MwRP5kVymzkVBQUH069eP48ePUyHdLOpPPfUUu3fvJjr9bPE3JSUlYWdnVyBHAZv1dyDsRhjLjyxnxv4Z7Di3I8MlbWc7Zzo/1pnRzUdTvkjWf6ERWbd2rZ7r+ZZu3fQj6u4257MQeYlcprZidevW5fr16+zcuROlFBMnTuQ///kPJUqUwNHRkas3H865ZMkSGjRogIuLC56ennTt2pUzZ87ccbwpU6bwxBNP4OzsTOHChWnWrBnbtm0DMr9MvWvXLlq1aoWXlxcuLi6UK1eOwYMHp+7P7DJ1UlISI0aMoGzZsjg4OFC2bFlGjBhBUlLaHNW3zjV58mRGjhxJ8eLF8fT0pH379pw7dy6bf4rZx9fVl1frvMpfr/xFwogElnZfSp+affD39CfOEsecA3Oo+ENFeizuwYbQDXIZ+xGdOKGfLHTrx5iSoifj6NtXP8Bg/nwJYpH/5ckwVuruS2BgWrvAwHu3Ta9Onbu3GzgwrV1OzCd78uRJbG1tKVSoEACff/45x44dIzAwkKVLl+Lk5MSkSZPo0qULVatWZdGiRUyePJmDBw/SrFkzrl+/nnqs9957j4EDB/LEE0+wYMECZs+eTdOmTTMNbYCYmBjatGmDra0tQUFBrF69mpEjR973O+y+ffvyxRdf0KdPH3777Tf69evHl19+Sd++fe9oO3bsWEJCQpg2bRoTJkxg+/bt9OzZ8xF+YrnH1saWTlU6MaPTDELfDmXva3tpX6k9SinmHZxHq1mt8PrKi47zOjLnnzmZz8MtMhUZqR/EUKEC9OypZ8oCqFED9u/X9w3LwxNEgWEYhilLnTp1jHv5999/77pP/w6d+TJ5clq7yZPv3Ta9J564e7tXX01rFxx8z7Lvafr06QZgHDlyxEhKSjKioqKMSZMmGTY2NkbHjh2NkydPGoBRu3ZtIyUlJfV9169fN9zd3Y1+/fplON7JkycNe3t749tvvzUMwzCOHz9u2NjYGMOGDbtrDbfOMX36dMMwDGPXrl0GYOzfv/++dZ88edIwDMM4cOCAARiffPJJhnZjxozJcKxb52ratGmGduPGjTMA4/z58/f8ed3r74DZQiJDjA/Wf2CU+KaEwShSl6o/VTWm7p6a4b+fuNPmzYZRtmza/2NOToaxaJHZVQmR84BgI5NMzJM943tFbPpe7MCB926b3u7dd2+Xvrddp86j11+lShXs7e0pUqQIgwcPpmfPnkybNi11f6dOnTJ8R7x9+3aio6Pp2bMnFosldfHz86NKlSps2bIFgA0bNpCSksLA9D+E+6hYsSKenp689tprzJ49m7Nn75ws43a3zterV68M22+93rx5c4bt7dq1y/C6+s3H2dytt54XlC9SnrEtx3J22FkODT7EFy2+oIhzEf4N/5cBKwfQLKgZf5/52+wyrY5hwMsvQ7Nm+tm+lSvDkSP6qUhduphdnRDmyZNhnNctXbqUXbt2ceTIEW7cuMHMmTMpUiTtvtbbRy2HhYUB0LJlS+zt7TMsBw4cIDJST2Zx608/P78s1+Lh4cGmTZsoUaIEgwcPpnTp0jz++OMsXrz4ru+Jujnl0e113ppLPOq2KZHSfzYAR0c9D3V8+lkb8igbZUNVn6oMbzKcM0PPMKHtBFzsXdh6ZitNpjehw9wOBF+Q53bfEh0N8+bp9ebNYedOHchCFHQSxiZ4/PHHCQgIoHLlyjhlMlXQ7SOnvby8AD2QateuXXcsgTe77t7e3gCcP3/+geqpVasWixcvJioqKvW2p27dunHw4MFM298K10uXLmXYfuv1rXoLGlcHV96q/xb/Dv6X1+u8jp2NHSuPraTulLq8uOhFrsVfM7tEU+zYoR9FCHriDm9vWLYMNm0Cd3dzaxPCWkgY5wGNGjXCzc2NkJAQAgIC7lgq3+xatGzZEhsbm9RwflB2dnY0aNCAMWPGkJKSwuHDhzNt16xZMwDm3eri3DTn5qNvmjZt+lDnzy/KeJbh5+d+5t/B/zKg9gAA5h+aT7WJ1fj9xO8mV5d7jhyBUqWgYUNYvjxte0gIdOxoXl1CWCOZ9CMPcHd3Z9y4cbzxxhuEh4fzzDPP4OHhwfnz59m8eTPNmzfnpZdeonz58gwbNozx48dz/fp1OnTogK2tLTt37qRKlSp07979jmP/9ttvBAYG0qlTJ/z9/blx4wbff/89bm5uNGzYMNN6qlWrRo8ePRg1ahQWi4VGjRqxfft2xowZQ48ePahRo0ZO/0jyhIpeFZnSYQpv1n+Tfsv7sefiHtrMbkOrcq14t+G7tC7fOl/eP56SAgsXQp8+kJiot6Ub8C/zRguRCQnjPOK1116jVKlSjBs3jl9//ZWkpCRKlixJ06ZNqVWrVmq7r7/+mgoVKjBx4kRmzJiBq6srNWrUoHXr1pket2LFijg7OzNmzBguXryIm5sbdevWZf369ff87nnGjBmUK1eOadOm8dlnn1GiRAmGDx/OJ598ku2fPa+rUbQG/zfg//h086d88dcXrA9dz/rQ9VTxrsKkdpNoVraZ2SVmm4gI8PHJuC00FPz9zalHiLxCZuASVis//h2Iiotiwo4J/LjrR6Li9EC3vjX78kuHX7C1sTW5ukdz5Ai0aQO3BsmPHg0ffqifKSyE0GQGLiGsQBHnIox+ajQn3jpB/9r9sVE2zNg/A6+vvPjiry+4kXjD7BIf2IUL+palcuXg1VehWjU9acfIkRLEQmSVhLEQJvB08mRqh6ks674Mf09/riVc48ONH1Lxh4r8cfIPs8vLsg8/hHr19HzSDg4wYgQcOKBn0RJCZJ2EsRAmal+5PSfeOsHCrgupWKQiF2Mu0mJmC56f/zxX46+aXd49tWoFX3wB58/rSTtuyYdj0oTIcRLGQphMKcULVV9g72t7+aDxBwAsO7KMchPK8c22b6xuvmvDgEmTYMMG/frtt6FzZ3NrEiKvkzAWwkq4OrgytuVYgl8NpnGpxlyJv8J769+j8/zOxFusY7YyiwWeegoGDdKv3dz0s4eFEI9GwlgIK1OnRB229tvKr51/xd3RneVHl1P1p6qmz3VtGFCmDNyaenzCBLhWMCcVEyLbSRgLYYWUUvSo3oO/X/kbf09/Tl49SZPpTZjzzxwTa9KTeYB+eMpbb8n3w0JkFwljIazY476Ps+/1fXSt2hWAXkt78cmmT4hNis2V8xsGrFoFN59VQv36cOWKvoVJCJF9JIyFsHLuju7Mf2E+wxoMA+DTLZ9S+tvSbAjdkKPntVigZk147jno3RsSEsDWFjw9c/S0QhRIEsZC5AFKKca3Gc+KF1dQ2qM0kXGRtJrVitazWnP22v2fQf0w3nlH3zMM0KAB2MnkuULkGAnjXBQUFIRSKnVxcHCgfPnyfPTRR6Y+2/fll1+mbNmyqa9PnTqFUoqgoCDTahKZu3Vf8kdNPsLF3oX1oesJmBLA1tNbs+0cFgt06wY//KBfz5unp7a0zduzdQph1SSMTbBw4UK2b9/OqlWraNOmDWPHjuX99983uyyRR9jZ2PF5i885NuQYT5Z+krAbYTQNakqHuR24EnflkY4dG6svSy9cqJ+utGABZPKwLyFENstSGCul2iqljiqlQpRSH2Sy30MptVIptV8pdUgp1S/7S80/atWqRYMGDWjVqhUTJ06kZcuW/PLLL6SkWNfkDsK6lXQvyeqeqxlafygu9i6sPLaSGpNqsPfi3oc+5oQJsG6dXp85E7p2zaZihRD3dN8wVkrZAj8BzwBVgR5Kqaq3NXsD+NcwjJpAc+AbpZRDNteabz3xxBPExcUREREBQGxsLMOHD8ff3x8HBwf8/f35/PPP7wjr8PBwBg8eTKlSpXB0dKRUqVL07t2bhIQEAEJCQujduzf+/v44OztTrlw5Bg0axJUrj9Z7EtajkEMhvm37LXtf20vFIhU5F32OZkHNOB55/KGO17cvDBgAhw5JEAuRm7IyJKMeEGIYRiiAUmoe0BH4N10bA3BT+knphYAowJLNtQKgRlvHjY3GJ9n36MlTp07h4eGBl5cXFouFNm3a8O+///Lxxx9TvXp1duzYwZgxY4iKiuKbb74B4MqVKzRq1IioqChGjBhBjRo1CAsLY/ny5SQmJuLo6MiFCxfw8/Pju+++o3DhwoSGhvK///2PZ599lu3bt2db/cJ8lbwqse/1fXSa14n1oetpOasli7stJqDEHU9qu8PatfqSdLNmUKIETJmSCwULITLIShiXBNIP1zwH1L+tzY/ACuAC4AZ0N4w7J9RVSg0EBgKULl36YerNF5KTk7FYLFy/fp2lS5eyePFivvvuO2xtbZk1axZ//fUXmzdvpmnTpgC0aNECgNGjRzN8+HB8fX359ttvCQ0NJTg4mNq1a6ceu0ePHqnrTZs2TT0GQKNGjahQoQJPPvkke/fuzfA+kfe52Lsw/4X5NJ7WmMMRh6k7pS7ft/2eN+u/edf3XL+ue8AxMXqu6Zt/1YQQuSwrYZxZV/T2bmEbYB/wNFAeWK+U2moYRnSGNxlGIBAIEBAQ8FBdy+zskZqlSpUqGV4PHjyYIUOGALB27VrKlClDo0aNsFjSLi60bt2aESNGsGPHDjp06MDvv/9O3bp17xmoiYmJfP3118ycOZPTp09nGLF99OhRCeN8qLBzYbb138aba95k9j+zeWvtW/x19i++b/s9RQsVzdDWMMDdPe118+a5W6sQIk1WBnCdA0qle+2H7gGn1w9YYmghwEmgCiJTS5cuZdeuXaxevZqWLVsyceJEZs6cCUBYWBinT5/G3t4+w1KvXj0AIiMjU//08/O753k+/PBDRo0aRa9evVi1ahU7d+5kyZIlAKbeSiVylqeTJ7Oen8VXLb/CzsaOBYcW4D/BnzXH12Ro98YbaeuHDsmtS0KYKSs9411ARaWUP3AeeBF46bY2Z4AWwFalVFGgMhCanYXmJ48//jgVKlQA4Omnn6ZGjRq8//77dOnSBS8vL/z9/VmwYEGm7711P7C3tzfnz5+/53nmzZtHnz59GDFiROq2mJiY7PkQwuq93/h9ulbryqsrX2VD6AY6zOvAiCdHMLLZSKZNU/z8s273wgtQ9fYhmUKIXHXfnrFhGBZgCLAOOAwsMAzjkFLqdaXU6zebjQEaKaUOABuB4YZhRORU0fmJo6Mj48aNIywsjIkTJ9K2bVvOnj1LoUKFCAgIuGPx9vYG9GXrnTt3sn///rseOzY2Fnt7+wzbpk+fnqOfR1iXsp5lWddrHUPqDsGSYmHU5lH0mfo/BgzQ+596Ku3hD0II82RpgjvDMFYDq2/bNind+gWgdfaWVnB06NCBunXr8vXXXxMSEsL06dNp0aIF7777LjVr1iQxMZETJ06wYsUKli1bhouLC8OGDePXX3+lZcuWjBgxgurVqxMREcHy5cuZNGkSbm5utG3blhkzZlC9enUqVKjAkiVL2LZtm9kfV+QyG2XDD8/+QPWi1Xntt9eYfWEEJYftpWnkTGZPdzG7PCEEWQxjkfM+++wz2rRpw9SpU1m3bh1ffPEFgYGBnDx5EldXV8qXL0+7du1wcNC3b3t6evL3338zYsQIvvjiCyIjIylatChPP/10apsffvgBwzD473//C8Czzz7L3LlzU79/FgXLKzVf5UbiDf77x38577GY0KrnCI9dfsfALiFE7lOGYc7o5ICAACM4OPiu+w8fPsxjjz2WixUJayN/B7JPSAhUrAh//AHFHj/MM3Oe4fS10xR1LcqGPht43Pdxs0sUokBQSu02DOOOCQBkbmoh8rmkJD2zFsDYsfCYz2Ns77+d+iXrc/nGZRr+0pDfjv1mbpFCFHASxkLkYwkJ0K4d3Boq8P33+s/ibsXZ2GcjTUo3ISYxhg5zOzB1z1TzChWigJMwFiKfSkmBli1h/Xr9evFiSD/fjKuDK5v6bmJk05EYGLy68lV+3vWzOcUKUcBJGAuRT73xBvz1l17/9Vfo3PnONnY2dox+ajRjW4wFYPDqwczaPysXqxRCgJWHsVmDy4T55L/9owkPh+XL9QMgFi+GdFOWZ+qDJh8wqtkoAPou68uoP0eRnJKc84UKIQArDmN7e3vi4uLMLkOYJC4u7o4JS0TW+fjoKS6XL8+8R5yZT5p/wnsN38PAYPTm0XRe0JkbiTdytlAhBGDFYezr68v58+eJjY2VXlIBYhgGsbGxnD9/Hl9fX7PLyZOibz6epXBhaP2AU/GMaz2O5S8ux9XelRVHV/D0zKeJiJXJ9ITIaVZ7nzFAdHQ0YWFhJCUl5VJVwhrY29vj6+uLe/pHCoks2bYNGjfWl6lvzpz6UA6GHaTdr+04c+0MJdxK8Huv36nmWy37ChWigLrbfcZWHcZCiKwLD4dbFxNGj4aRIx/teGevnaX7ou5sP7cdbxdvfm73My9UfeHRCxWiAJNJP4TIxyyWtCAGGDbs0Y9ZyqMU63uvp1GpRkTERtB1YVem7J7y6AcWQtxBwliIPM4w0mbYcnGB06fBzS17ju3q4Mrmlzfzn0b/AeC1315jyeEl2XNwIUQqCWMh8rhZs/R9xAAzZ0Lp0tl7fF9O+3EAACAASURBVDsbO75s9WXq5CC9l/ZmXci67D2JEAWchLEQeZhhwMqVen3qVOjSJefONar5KF6u9TKxSbG0+7UdE3dNlDsdhMgm8ghFIfIwpaB+fbh2DV55JafPpZjSfgo22DBt3zTeWP0Gh8MP88OzP+TsiYUoAKRnLEQeN2AArF2rgzmn2dnYMbXDVGY/PxtHW0d+3PUjM/fPzPkTC5HPSRgLkQfNnQuffgrx8eDpCTa5+H+yUoqeNXrywzO6R/zG6je4HHM59woQIh+SMBYijzl1Cl56CT75BJYuNa+OAU8MoE35NsQkxvDab6/J98dCPAIJYyHykIQEeO45vV68OHTvbl4tSikmtJ2Au6M7y48uZ9reaeYVI0QeJ2EsRB4yZox+AETZsvDPP7l7eTozlb0r822bbwF4c82b7Dq/y9yChMijJIyFyCO2bIH//U8H8IwZjzb3dHbqV6sffWv2Jc4SR88lPbmecN3skoTIcySMhcgDrlyBXr30fcUffghNm5pdURqlFIHtA6nmU43jUcd5bu5zxCTGmF2WEHmKhLEQeYCtLTRvru8p/uQTs6u5k4OtAyt6rKB4oeJsOb2FzvM7k5icaHZZQuQZEsZC5AHu7nqqyw0bwN7e7GoyV65wOf58+U98XX1ZH7qeFxe9SFKyPP5UiKyQMBbCioWH6+WWQoXMqyUrKnlV4rcev+Fs58zSI0sZsnqI3PIkRBZIGAthpQwDBg6EatVg82azq8m6uiXrsrHPRpzsnAjcE8jozaPNLkkIqydhLISVmj8fli3Ts2z5+5tdzYNpWKohczrPAWD05tHM2j/L5IqEsG4SxkJYoVOnYNAgvf7NN9n/WMTc0Pmxzox4cgQAfZb1YfSfo+WStRB3IWEshJWxWKBdO7h6FVq21A+CyKs+fepTvmvzHQCjNo/iqRlPEZ0QbXJVQlgfCWMhrEy/fvDvv1CiBAQF5c7TmHKKUoq3G7zNwq4LcbJzYvPpzby46EVSjBSzSxPCqkgYC2FFLl6EhQv1+rRpULKkufVklxeqvsCBQQfwcPRgTcga+i7riyXFYnZZQlgNCWMhrEjx4nrk9OuvQ5s2ZleTvSoUqcD8F+bjYu/C7H9m8/aat80uSQirIWEshJWpXx9+/tnsKnJGmwptWP7icmyVLRODJzJ1z1SzSxLCKmQpjJVSbZVSR5VSIUqpD+7SprlSap9S6pBSKg/dFSmE+UJD9ahpSwG4ctuyXEsmPTcJgEGrBvHnqT/NLUgIK3DfMFZK2QI/Ac8AVYEeSqmqt7XxBCYCHQzDqAZ0zYFahciXDAPKl4f33oPRBWR+jAFPDOCdBu9gSbHQZUEXgi8Em12SEKbKSs+4HhBiGEaoYRiJwDyg421tXgKWGIZxBsAwjLDsLVOI/Gvu3LT1t94yr47c9lWrr2hboS1RcVF0WdCFK3FXzC5JCNNkJYxLAmfTvT53c1t6lYDCSqk/lVK7lVJ9MjuQUmqgUipYKRUcnn7CXSEKqOhoePddvT50KPj4mFtPbrK1sWVxt8XUKFqDM9fOUGNSDUKvhJpdlhCmyEoYZ3aX4+3T6NgBdYB2QBvgY6VUpTveZBiBhmEEGIYR4FOQ/tURIhOGoe8pvnQJatfW3xkXNC72LizrvoyqPlU5F32OtrN1T1mIgiYrYXwOKJXutR9wIZM2aw3DuGEYRgSwBaiZPSUKkT99+y0sWQIeHrBgAdgU0Hsb/Av7s/nlzZQvXJ7jUcd5asZTEsiiwMnK//67gIpKKX+llAPwIrDitjbLgSeVUnZKKRegPnA4e0sVIn85dEj/OWMGVKhgbi1m83bxZkOfDVTyqsQ/l//hnXXvyDzWokC5bxgbhmEBhgDr0AG7wDCMQ0qp15VSr99scxhYC/wD7ASmGoZxMOfKFiLv++UX2LkTOt4+HLKAKutZlkVdF2FvY8+M/TPk0YuiQFFm/fYZEBBgBAfL7QxCiIwW/7uYbou6kWKkMLX9VPo/0d/skoTINkqp3YZhBNy+vYB+SyWEOcLCoEMHWL9eD+ASd+pStQuT2ulJQYauG0pIVIjJFQmR8ySMhcglhgF9+sDKlTB2rNnVWLcBTwyg82OdiUmMoc3sNkTGRppdkhA5SsJYiFwycyasWwdeXno9Lz8aMacppfjp2Z8o5V6K0CuhdJjXgdikWLPLEiLHSBgLkQsiI/V0lwDjx4Ofn7n15AXFChVje//tlHIvxbaz2xi5aaTZJQmRYySMhcgFH3wAERHQvDn07m12NXlHSfeSzHp+FgDjt49n+9ntJlckRM6QMBYih82cCVOngr29fjSiXJ5+MM3KNuM/jf6DgUG3Rd24FHPJ7JKEyHYSxkLkMDc3KF4cPv0UqlQxu5q8aczTY2hcqjHnos/x/PznORd9zuyShMhWEsZC5LAOHfQI6uHDza4k73KwdWBxt8WUdCvJjnM7qDmpJrvO7zK7LCGyjYSxEDkk6ub0yra2UKeOXJ5+VEULFWXHgB20KteKqLgonv31WblkLfINCWMhcsCxY+DvDyNHQkqK2dXkH37ufqx6aRWNSjUiIjaCZ+Y8Q4IlweyyhHhkEsZCZLOUFP1oxOhoOHGi4D6NKafY29qzpNsSyhUux75L+xi6dqg8VELkefLPhBDZ7JtvYNs2KFYMfvrJ7Gryp6KFijKvyzxslS2Tdk/irTVvSSCLPE3CWIhs9Pvv8OGHen3yZPD0NLee/Kxuybos6rYIB1sHftz1IxP+b4LZJQnx0CSMhcgmV65A376QnAxDh+pR1CJndarSKfWhEu/9/h6rjq0yuSIhHo6EsRDZZPhwuHRJj5z+4guzqyk4Xq71Mv1r9yfZSOaFhS+w49wOs0sS4oFJGAuRTcaMgRdegPnzwdHR7GoKDqUUge0DefHxF4m3xNNlQRf2XNxjdllCPBAJYyGySdGisHAhlC9vdiUFj42yIahjEI1LNebC9Qu0mNmCE1EnzC5LiCyTMBbiEUREQPXqcPmy2ZUIRztH1vRcQ/Oyzbkaf5Xui7qTlJxkdllCZImEsRCPYOBAOHgQ3nzT7EoEgJujG0u6LaGMRxl2X9zNx5s+NrskIbJEwliIh7R6NSxdqtfff9/cWkSaws6FmdFpBgrFl39/ydbTW80uSYj7kjAW4iGcPw/t2un1L7+EunXNrUdk1KxsMz5o8gEAQ9YMIS4pzuSKhLg3CWMhHpBhQNOmer11a3j3XXPrEZkb1mAYRV2L8s/lf+izrI/MYS2smoSxEA9owgQIDdXr06frpzIJ6+Pj6sOS7ktwsnNi0b+L6LygM/GWeLPLEiJTEsZCPICUFPi//9PrCxdCiRLm1iPurVGpRmzttxVvF29WH1/N8/Ofl0AWVknCWIgHYGMDc+fCkSN6gg9h/QJKBPBn3z/xcfFhbchamkxrwrX4a2aXJUQGEsZCZIFhwNWraa8rVzavFvHgqvlWY2OfjZT2KM3ui7t5bu5z0kMWVkXCWIgsmDIFatWCQ4fMrkQ8rOpFq7Pl5S2UdCvJX2f+4pXlr8hjF4XVkDAW4j727YO334bTp/W6yLvKeJZh1UurcHNwY+7BufLYRWE1JIyFuIerV6FLF4iPh1degZdeMrsi8ahqFqtJUKcgQD92cfvZ7eYWJAQSxkLcVXIy9O6tb2OqXRt+/BGUMrsqkR06P9aZl2u9TLKRTNs5bfk3/F+zSxIFnISxEHcxZgz89hsUKQLz5oGzs9kViewU+Fwgnap0IjohmqdnPM0/l/8xuyRRgEkYC5GJkyfh8891T3j+fKhUyeyKRHazt7Vn1vOzeKrsU1y+cZmWM1sSeiXU7LJEASVhLEQm/P1hxgz4+mto2dLsakROKeRQiJU9VtK0TFPCY8NpFtSMoxFHzS5LFEASxkLcxUsvwTvvmF2FyGmuDq4s6baEgBIBnIs+R/u57Qm/EW52WaKAkTAW4qbEROjaFbbKE/cKHC8XLzb22UgV7yocjzrOoFWD5B5kkauyFMZKqbZKqaNKqRCl1Af3aFdXKZWslJKJAkWeYhgwaBAsWqRHUCcmml2RyG3uju6p9yAvPryYTvM7kWKkmF2WKCDuG8ZKKVvgJ+AZoCrQQylV9S7tvgTWZXeRQuS0ceNg2jQ9YnrhQnBwMLsiYYZyhcsxveN0AFYcXcF3O74zuSJRUGSlZ1wPCDEMI9QwjERgHtAxk3ZvAouBsGysT4gcN3s2DB+u16dPh7p1za1HmKtL1S782vlXAD7e9DGHww+bXJEoCLISxiWBs+len7u5LZVSqiTwPDDpXgdSSg1USgUrpYLDw2WAhDDfnj0wcKBeHzsWunc3tx5hHXpU70GvGr2ITYrlubnPcSnmktkliXwuK2Gc2ZxDt49s+A4YbhhG8r0OZBhGoGEYAYZhBPj4+GS1RiFyREwMdOoEcXHw4otpvWMhAH585kdqF6tN6JVQ+i3vR1JyktkliXwsK2F8DiiV7rUfcOG2NgHAPKXUKeAFYKJSqlO2VChEDilUCIKCdG84KEimuhQZeTh5MO+Febg7urM2ZC3//eO/Zpck8rGshPEuoKJSyl8p5QC8CKxI38AwDH/DMMoahlEWWAQMNgxjWbZXK0Q2e/ppPdWlo6PZlQhrVMmrEit7rARg3LZxBO4ONLkikV/dN4wNw7AAQ9CjpA8DCwzDOKSUel0p9XpOFyhEdps5U885LURWNC3TlG9afwPA67+9zpTdU0yuSORHyqwb2wMCAozg4GBTzi0Krk2boE0bsFjgn3/g8cfNrkjkFaP/HM2ozaMAmNN5Di9Vl+dpigenlNptGEbA7dtlBi5RYKxbB888A0lJ8PbbEsTiwXzS/BM+bvoxAP2W92P+wfkmVyTyEwljUSDs2wdt20JCgh6w9c03Zlck8qJRzUfxVr23SExOpNfSXmw7u83skkQ+IWEs8r3z56F2bb3esKGe5MNG/uaLh2CjbPiu7Xe8Xud1LCkWnvv1OQ6FHTK7LJEPyD9JIl9LSdH3EgPUqwd//AF2dubWJPI2pRTj24yndrHaXIm/QqtZrTgYdtDsskQeJ2Es8jUbGwgM1N8PL10KTk5mVyTyA2d7Z7b020KzMs24GHORDnM7yGMXxSORMBb5Xu3acOAAlChhdiUiPynkUIi1vdYSUCKAk1dP0m95P3nsonhoEsYi3zEMeO89mDjR7EpEfudk58TS7kvxdPJk1fFVzPpnltkliTxKwljkO6NG6dHSw4bByZNmVyPyOz93Pya0nQDAsHXDiIiNMLkikRdJGIt85bvv4NNP9XfFs2eDv7/ZFYmCoHeN3jzt/zRRcVH0WtJLHiohHpiEscgXUlLgo490bxhg2jTo2tXcmkTBoZTilw6/4OHowboT6/j+/743uySRx0gYizzPYoG+ffXziG1tYfJk/VqI3FTWsyzTOk4D4P3177Pq2CqTKxJ5iYSxyPPOntX3D7u6wpo1MHCg2RWJgqrzY535uOnHGBh0XdiVaXunyQhrkSUSxiLP8/eHbdt0ILdqZXY1oqAb1XwUA2oPIM4SR/8V/Xnv9/ckkMV9SRiLPOnSJZg0Ke11mTJ6hi0hzGajbAhsH8i0DtOws7Fj/I7xTAqedP83igJNJgYUeU5oqO4Bh4bqGbVeftnsioTISClFv9r9iIyL5P317zN49WCiE6IZ3mS42aUJKyU9Y5Gn/PMPNG6sg7hOHWjXzuyKhLi7dxu+yyfNPgHgoz8+Yl3IOpMrEtZKwljkGWvWQIMG+hL100/Dpk3g42N2VULcnVKKUc1H8UHjD0gxUmg/tz0LDy00uyxhhSSMRZ4wdSo8+yzExcHzz8Pq1eDmZnZVQmTN5y0+Z1iDYSSlJNF3WV/2XtxrdknCykgYC6sXHw/ffqvXe/aE+fPB0dHcmoR4EDbKhvFtxtOtWjfiLHE8Of1J1p9Yb3ZZwopIGAur5+QEK1ZAUJCe4tLe3uyKhHg4MzrN4KXqL3Ej6QYd53Vk9fHVZpckrISEsbBK27bpqS1v3Z5ZvrzMqiXyPic7J2Z2mknfmn2Js8TRcV5HVhxdYXZZwgpIGAurYhjwww/QrJl+6MOiRWZXJET2srWxZXrH6QyoPQBLioUXFrzA0sNLzS5LmEzCWFiNGzegVy946y093/SwYdCpk9lVCZH9lFIEtg+kf+3+JKUk0XlBZ0b/OdrssoSJJIyFVTh+XN+29Ouveo7p+fNh/Hj5fljkX7cC+etWX6NQjNo8igErBsjUmQWUhLEw3YEDEBAABw9C5cqwcyd062Z2VULkPBtlw7uN3iWwfSAKxS97f2Hy7slmlyVMIGEsTFe5MpQuDV266CCuWtXsioTIXQOeGMCcznMAGLp2KMuOLDO5IpHbJIxFrjMMmDcPIiL0awcHWL8eFi4Ed3dzaxPCLD2q96B3jd4kJCfw/PznGbxqsFyyLkAkjEWuOn8eOnaEHj1g6NC07cWKgVLm1SWENfilwy+Mbz0eheLn4J9pP7c9NxJvmF2WyAUSxiJXGAZMmwbVqsHKleDhoeeXll/8hUhjb2vPsIbDWNljJS72Lqw6vorn5j7H9YTrZpcmcpiEschxZ85A27bQvz9cuwbt28OhQ/DKK9IbFiIz7Sq1Y0f/HRQrVIw/T/1J69mtiUmMMbsskYMkjEWOunoVqleH338HLy9969Ly5VCypNmVCWHdqhetztZ+WynhVoId53bQZFoTjkQcMbsskUMkjEWOuHX52dMTunaFF17Qty716CG9YSGyqkKRCvzR5w+KFSrG/sv7qTmpJmtD1ppdlsgBEsYiWx0+DK1b65HRt0ycqF8XK2ZeXULkVZW9K3No8CG6Vu1KYnIi7ee2Z+qeqTLSOp+RMBbZIiYGRo6EWrX0bUrff5+2z8HBvLqEyA+KOBdhbpe5vFnvTSwpFl5d+Sqv/fYaySnJZpcmskmWwlgp1VYpdVQpFaKU+iCT/T2VUv/cXLYppWpmf6nCGl27BmPHgr8/jBkDiYl6oNby5WZXJkT+Ymtjy4S2EwjqGISDrQNT9kyh68KuWFIsZpcmssF9w1gpZQv8BDwDVAV6KKVunyPpJNDMMIwawBggMLsLFdYnOFiH8Ecf6Qk8GjSArVth6lQ9WEsIkb2UUvSt1ZfF3RbjaOvI0iNLCQgMYOXRlWaXJh5RVnrG9YAQwzBCDcNIBOYBHdM3MAxjm2EYV26+3AH4ZW+ZwlrExqatP/44ODtD06Z6tPS2bdCkiXm1CVFQPFfpOTb13URJt5Lsv7yfDvM60GtJLyJiI8wuTTykrIRxSeBsutfnbm67m/7Amsx2KKUGKqWClVLB4eHhWa9SmO7gQejXD0qVgis3f+1yctK94z//hFatZJS0ELmpYamGhLwVwlctv8LR1pE5B+ZQ9aeqbDm9xezSxEPIShhn9k9spsP4lFJPocN4eGb7DcMINAwjwDCMAB8fn6xXKUxhGPDHH/Dss/pe4aAgfd/wpk1pbYoXlxAWwixOdk683/h9Dgw6QLMyzQiPDadZUDOm7Z1mdmniAWUljM8BpdK99gMu3N5IKVUDmAp0NAwjMnvKE2YwDJgzRz/WsEULWLNGX45+4w04dgw6dza7QiFEehW9KrKxz0aG1B0CQP8V/RnxxwjiLfEmVyayKithvAuoqJTyV0o5AC8CK9I3UEqVBpYAvQ3DOJb9ZYrcpJQehLVnD/j66lHSZ8/Cjz9C+fJmVyeEyIytjS0/PPsDo5uPBuDzrZ9Tc1JNNp/abHJlIivuG8aGYViAIcA64DCwwDCMQ0qp15VSr99sNhLwAiYqpfYppYJzrGKRra5fh19+gaeegu3b07Z//DEEBsLp0zBihIyOFiKvGNlsJH/1+4vHvB/jWOQxms9ozn/W/4eL1y+aXZq4B2XWLC4BAQFGcLBkthni4mDtWli8GJYuTRsh/eqrOoCFEHlfgiWBz7Z8xmdbPwP098ujm4/m/Ubvo2Sgh2mUUrsNwwi4fbvMwFXAvPMO+Pjo733nzNFB3LSp7h2PG2d2dUKI7OJo58iYp8ewo/8OWpVrRbwlnuEbhtPu13acvnra7PLEbSSM87ELF2DWLLh8OW2bUnDjhh6cNXYsnDgBmzfrxxl6eJhXqxAiZ9T3q8/vvX9nUddF2Cgb1oSsofG0xuy/tN/s0kQ6cpk6H4mN1RNvrF+vl7179faJE2HQIL1+7hxYLFC2rGllCiFMcizyGB3ndeRIxBFc7V35oMkHvN/ofRztHM0urcCQy9T5XNu2+nGFrVrBV1/pIHZy0vcIFy+e1s7PT4JYiIKqklcldg/cTY/He3Aj6QYfb/qYp2c+zbFIuQnGbBLGecjZs7BgAbz5pr7MnJCQcb/FArVrw3vv6Z5xVBSsWgWdOplTrxDC+rjYuzCn8xzmdplLCbcSbDu7jao/VeWzLZ/JYxlNJJeprdilSzB9Ouzfr287OnMm4/6//oLGjfV6aKi+/Ui+9xVCZNX56POM3DSSoP1BpBgp1CtZjy9bfknzss3NLi3futtlagljk8XH61mtDh2CAwf0JBtDh+p9oaEZJ9nw8ICGDfXSvDnUrw+O8lWPEOIRrTi6gn7L+xEVFwVA16pdGdlsJI/7Pm5yZfmPhLEVmTNH3+N76BCEhEBKStq+2rX1zFegp6X8z3+gShWoVw+qVgVbW3NqFkLkbzGJMXz515eM2zaOhOQEFIpBAYMY13ocLvYuZpeXb0gY54Jr13RvNiRELydO6OX4cZg3L+3xgsOH60FWADY2uvdbrZp+JGGdOvIdrxDCPOejz/PZls8I3BNIipFCFe8qfNHiC9pXbo+NkmFGj0rC+BElJ0NEhB5EdeaMXtzcoH9/vf/yZShW7O7vnzZNP4IQdM/3yBEdwJUr61HPQghhTfZd2sdLi1/icMRhAJqWaUpQxyD8C/ubXFneJmGciVsBe/ly2nLpEvTsmXY70MiReorI8PCMl5MBataEffv0umHo73uLFoVy5aBiRd3jLV9er5cuDXZ2ufv5hBDiUcQmxTI5eDKfbf2MqLgo7Gzs6F2jN8MaDKN60epml5cn3S2M8108WCw6YC9dyhiyZcpAt266zfHj+pJxRMSdAQvwxBNpYZyYmDaDVZEiOlRvLVWqpL1HKQgLk2f7CiHyDxd7F4Y1HEaP6j0YtGoQy44sY/q+6UzfN50PGn/Ax80+lu+Ts4nV94zj4vSTg65e1cuVKxAZqf+MioJPP9WXiwG6dtUDozL7SO3awW+/6fWwMN2DBfD21uvpl1df1YOlbrVNStLzOTs4ZMMHF0KIPCokKoTx28fzc/DPAHi7eNOvVj9eqf0KVbyr3OfdAqzwMrWfX4DRsWMw0dHcsfTurS8Pg568onXrux/nyBH9vSvo982Zo++3LVYsY8DWqgV9+uh2KSm65+zjA/b2Ofs5hRAiv/nz1J+8s+4d9l7am7qtgV8DRjcfTevy9/gHW1hfGDs7Bxjx8Zn3jN94Qz/IHvSEF926QeHCerpHDw99udjLS2/r1SutlxsTowdDyXezQgiRswzD4M9TfzJj/wyWHVnGtYRrALQs15I3671Jh8odTK7QOlldGPv7BxjvvhuMuzt3LD4+OmiFEEJYv+sJ1/lp1098uPHD1G1dq3bl2zbfUtK9pImVWR+rC2NrGE0thBAi+5y5doape6by9bavibPE4WDrQNeqXRkUMIhGpRqhZISrhLEQQojccebaGd5e+zbLjyzHQGdMdd/qDKk3hN41euNs72xyheaRMBZCCJGrTl09ReDuQKbsmUJEbAQAPi4+9K3ZlyH1hlDGs4zJFeY+CWMhhBCmiLfEM/fAXH7c9SN7Lu5J3V6neB2GNhhKz+o9C8wlbAljIYQQpjIMg53nd/Ld/33H8iPLibPEAXqw18A6A2lcqnG+v4QtYSyEEMJqxFviGbt1LGP/GktSShIAjraONCzVkJb+LWlWthkN/BpgZ5O/7lWVMBZCCGF1Docf5ufgn9lyegv7L+/PsM/X1ZeW5VrSuUpn2lZoi6uDq0lVZh8JYyGEEFYtMjaSP0/9yfrQ9aw7sY5TV0+l7nO2c6Z3jd4MqTckTz+kQsJYCCFEnpFipHA04ii/HfuNpUeWsv3c9tR9dUvU5ZXar/Di4y/i6eRpYpUPTsJYCCFEnhV8IZgpu6cw/9D81Kk3neyc6Fi5I71r9OZp/6fzxOAvCWMhhBB5XlxSHEuPLGXa3mlsPLkxdbuTnRNPlX2KdhXb0a1aN3xcfUys8u4kjIUQQuQrJ6JOMH3fdNaErMlw/7KNsqF2sdo0Kd2E7tW607BUQxOrzEjCWAghRL51KeYS60LW8evBX9kYupFkIzl1n5ezF/6F/anhW4PGpRtTt0RdHvN5zJTbpiSMhRBCFAgxiTHsPL+TRf8uYtY/s4hJjLmjjaOtI37ufpRwK0G9kvV4svSTVPauTPnC5bG3zbkH3UsYCyGEKHBSjBQux1zmWOQxdp7fya4Luwi+EMzJqyczbW+rbKniXYWOlTtSxbsKpT1KU9qjNGU9y2bLlJ0SxkIIIcRN0QnRXLx+kVNXT/HHyT/Yf3k/RyOPcvrq6dQnTaXn5exFhSIVKOtZlkpelShfuDxVfapSyasSHk4eWT6vhLEQQghxH3FJcWw5vYU/Tv7BmegznLl2huORxwmPDb/re3xcfChaqCi+rr74uvri5eylFxcvvF28KV6oOD6uPnrdrXimYZy/Jv0UQgghHoGzvTNtKrShTYU2qdsMw+D89fOcunqKg2EH2XtxLxdjLnLm2hmORh4lPDb8nmGdFRLGQgghxD0opfBz98PP3Y8mpZtk2JeckszlG5cJvxFO2I0wwm6EERkXSWRsJFFxUVyMucjlG5eJjI0kMi6SMMIyPUeWwlgp1RaYANgCUw3D+OK2/erm/meBWOBlwzD23HEgIYQQIh+xtbGlhFsJSriVyFJ79X7mg8Bs7vtGpWyBn4BngKpAD6VU1duaPQNUZd1uTgAACfFJREFUvLkMBH7OUlVCCCGEuH8YA/WAEMMwQg3DSATmAR1va9MRmGloOwBPpVTxbK5VCCGEyJeycpm6JHA23etzQP0stCkJXEzfSCk1EN1zBohRSh19oGqzlzcQYeL5zSafv+B+/oL82UE+v3x+cz9/mcw2ZiWMM7vAffv9UFlpg2EYgUBgFs6Z45RSwZkNLy8o5PMX3M9fkD87yOeXz2+dnz8rl6nPAaXSvfYDLjxEGyGEEEJkIithvAuoqJTyV0o5AC8CK25rswLoo7QGwDXDMC7efiAhhBBC3Om+l6kNw7AopYYA69C3Nk0zDOOQUur1m/snAavRtzWFoG9t6pdzJWcbq7hcbiL5/AVXQf7sIJ9fPr8VMm06TCGEEEJoWblMLYQQQogcJGEshBBCmCzPh7FS6k2l1FGl1CGl1Ffptn+olAq5ua9Nuu11lFIHbu77/uZUniilHJVS829u/z+lVNl07+mrlDp+c+mbbrv/zbbHb77XIXc+dUZKqfeUUsb/t3f+MVJVVxz/fMMWSFVkgSJbpVX+kKhtCkTbbgW7VWvohpI0bdM1MWklTeuPmP5Iilj6g2o1AVKDTZNCYwLaXY2IUlJCC2LAH9WilIJtCquga0XBVpG2SqQqp3/cM9m3w8wu02Xn7cycT/Ly7j333jf33Pdmz7xz754raUJGVtf6S1oqaY+kZyStlTQ2U1bXug8GSbN9XPZKWpB3fypB0mRJWyTt9u/7t1w+TtJDfi8ektScaTPkz0I1kTRC0p8lrfd8I+k+VtIa/97vltRaV/qbWc0ewGeAzcAoz0/08/nALmAUcA6wDxjhZU8BraT/jf4d8DmXXwcs93QHcJ+nxwHP+7nZ081ethro8PRy4NocxmAyaXHdi8CERtEfuAJo8vRiYHGj6D6IMRvh4zEFGOnjdH7e/aqg/y3ADE+fBjzr93sJsMDlC6r9LFR5DL4L3AOs93wj6X4X8HVPjwTG1pP+uX/BBnlzVgOXl5DfBNyUyW/0wW8B9mTkVwIrsnU83USK0KJsHS9b4TJ5nYJBaAU25jAGa4CPAT30GuOG0d8/+wtAVyPqXuE49eln8VjV2gGsAz4LdAMtLmsBuqv1LFRZ37OAh4FL6TXGjaL7GOAFfNFxRl43+te6m/pcYJa7FB6RdJHLy4XnPNPTxfI+bczsXeBfwPh+rjUeOOx1i69VFSTNBV42s11FRQ2hf4Z5pF+40Hi6V0I5fWoOdyFOB7YBZ5jHNfDzRK9WjWehmiwD5gPHMrJG0X0K8E9gpbvp75R0CnWk/7Dfz1jSZmBSiaKFpP43A58ELgJWS5pC+fCc/YXtrLTNCYUAHSwD6P99krv2uGYlZDWnf3+6m9k6r7MQeBfoKjQr06+a0n2IqNV+90HSqcADwLfN7N8+5VeyagnZyX4WqoKkOcA/zOxPktpOpEkJWU3q7jQBM4AbzGybpDtIbuly1Jz+w94Ym9nl5cokXQs8aMl38JSkY6Qg4OXCc+73dLGcTJv9kpqA04FDLm8rarOV5MIYK6nJf0UNSQjQcvpL+ihpLmSX/zE6C9gh6ePUif793XtIiyqAOcBl/gxAneg+RNR82FpJ7yMZ4i4ze9DFr0pqMbMDSrvFFXZvr8azUC0uBuZKagdGA2MkddIYuhf6tt/Mtnl+DckY14/+1fT7D8E8wjXAzZ4+l+RKEHABfSfvn6d38v5p0pt0YfK+3eXX03fyfrWnx5HmKpr9eAEY52X303cRz3U5jkUPvXPGda8/MBv4G/CBInnd6z6IMWvy8TiH3gVcF+Tdrwr6L+BuYFmRfCl9F/EsqeazkMM4tNE7Z9wwugOPAVM9vch1rxv9c/+CDfLmjAQ6gb8CO4BLM2ULSSvouvHVci6/0OvvA35BbxSy0aQ/sHtJq+2mZNrMc/le4OqMfIrX3ettR+U4Fj24MW4E/f0zXwJ2+rG8UXQf5Li1k1Yh7yO5+3PvUwV9n0lyDz6Tue/tpHm9h4Hn/Dwu02bIn4UcxqGNXmPcMLoD04Dtfv9/QzKMdaN/hMMMgiAIgpyp9dXUQRAEQVDzhDEOgiAIgpwJYxwEQRAEORPGOAiCIAhyJoxxEARBEORMGOMgGAKUdtEa6Ojxuqsk7R/gklVBUo8HkziZ11t1AvVWFcYjCBqRYR+BKwhqlNai/FpSEIJFGdnRqvUmCIJhTRjjIBgCzOyP2byko8BrxfLBImmUmYVRD4IaJ9zUQTBMkDRd0mOSjvgm5tcUlX/N3duXSLpf0mHSrkVIavLN1PdIOirpFUk/kzQ6075J0i2S9kl6W9Jrkh6XNLNEXzp8A/e3JG0vU+cqSbsy1/q1xwceSM/LJO3wdvskffP/GrAgqCPizTgIhgdjSJvGLwNuBq4Gfimp28y2FNXtAu4FvkTvd7gT+DywGHgCOA+4BTgb+KLXuRH4DilM4E7/zAtJsXezzAKmAj8E3vbrrJd0tpkdBpD0DdK+rveR9o79IHAb8AlJM8zszVJKSjoP2EAKa9hBih28CDgVeG/AUQqCOiWMcRAMD04jbTaxBUDSo6TtMa8Eio3xGjObX8hImgV8Bfiqmd3t4s2SDgGdkqaZ2U7SPPYmM7sjc63flujLGGCamb3h1z9ICq7fDtwjaQTJQG81s45MP/aQgvnPA35eRs8fAP8BrjCzt7zdE6Q4wTW1g1QQnEzCTR0Ew4Mj2Tdgnwd+DvhQibpri/Kzgf8CD7grusm3gNvk5Zf4+WmgXdKtkmZKGlmmL08WDLHzFz8X+jKVtIl7V7aRmT0OvAh8upySpB8EGwqG2Nu9BPyhnzZBUPeEMQ6C4cEbJWRHSTvJFHOgKD+RtIPZm8A7maOwt+t4P98G/BiYS3qDfV3SSkkTiq53KJvJLBAr9KXg1i7uB8BBjnd7Z2kBXi0hLyULgoYh3NRBUHsUb7X2Omlud1aZ+q8AmNk7pDnlxZImAXOA24H3k9zcJ0rBWE8qUTaJNB9cjgPAGSXkpWRB0DDEm3EQ1D6/J721nm5m20scx83FmtlBM7sT2Ax8pMLP6ya9yXZkhZI+BXwYeKSftk+SXOWnZNpNBi6usA9BUFfEm3EQ1DhmtlXSvcAaSbeTNkY/RlpJ3Q7caGbPSlpHCjyyg+QWn06ab15R4ee9J+lHwAqP1tUJnAncSprnXtlP858CXwY2SVpKcq//hHBTBw1OGOMgqA+uAm4grWReSJpv7gE20mvoHiUZwutJrum/A0tIRrQizOxXko4A3wPWkearNwDzy/1bk7fbLakdWEr6t6iXSa7zVqCt0n4EQb0gs+LppyAIgiAIqknMGQdBEARBzoQxDoIgCIKcCWMcBEEQBDkTxjgIgiAIciaMcRAEQRDkTBjjIAiCIMiZMMZBEARBkDNhjIMgCIIgZ/4HOLmhNSN5j2IAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 576x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 使用matplotlib绘制精度和召回相对于阀值的函数图\n",
    "\n",
    "def plot_precision_recall_vs_threshold(precisions, recalls, thresholds):\n",
    "    plt.plot(thresholds, precisions[:-1], \"b--\", label=\"Precision\", linewidth=2)\n",
    "    plt.plot(thresholds, recalls[:-1], \"g-\", label=\"Recall\", linewidth=2)\n",
    "    plt.xlabel(\"Threshold\", fontsize=16)\n",
    "    plt.legend(loc=\"upper left\", fontsize=16)\n",
    "    plt.ylim([0, 1])\n",
    "\n",
    "plt.figure(figsize=(8, 4))\n",
    "plot_precision_recall_vs_threshold(precisions, recalls, thresholds)\n",
    "plt.xlim([-700000, 700000])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAf0AAAF5CAYAAACV7fNGAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3deZgU5bn+8fsZdkFhEARkR9BoUBTHBY8LiBCXqCfGBTWKW/AkRqMnMSKiQeMS9RxzfscEI8YTdxETNYsm4oaiYhSiuMWNRRFckAGRRVnm+f3x9lg9wyw9M91VvXw/19VXvVVd3fVQzuXdVfXWW+buAgAAxa8s6QIAAEA8CH0AAEoEoQ8AQIkg9AEAKBGEPgAAJYLQBwCgRBD6AACUiNhD38x6mNnsRta51czmmNnkuOoCAKDYxRr6ZlYu6XZJHRtY5xhJrdx9hKRBZjYkrvoAAChmcR/pb5Z0gqTVDawzUtKMVHumpP1zXBMAACWhdZwbc/fVkmRmDa3WUdLSVLtS0vDaK5jZBEkTJKl16/I9t9lm0NfvVVaG6cCBUteuWSgaAIA8M2/evM/cvXtTPxdr6GdojaQOqXYn1XE2wt2nSZomSRUVFT537tyv3zvxRGn6dOmqq0L7nXekRx+VDj9c2mGHLTe2YoX02mvhtXy5dM45Uo8eOfhXAQCQJWb2fnM+l4+hP0/hlP4LkoZJers5X/L889Jvfys980yYf+658GOgslJ65BHpySfD6/1au+3WW6W775ZGjgzzVVXSggXS669LCxdKixeH16JF4dWjR3i/+uSFu/TZZ1K7dtI22zSncgAAciPR0DezXSSd5O7pvfQfkjTbzLaXdJikfZvynZs2hemvf11z+YMPSsccI/31r9LGjdHyjh2loUOlf/wjzC9bJo0aJZ11lvTqqyHs162rf3uLFkl9+kh77hn9EFi7Nrz38stSr15h2eLF0vr10rHHSmVlYbvVqqqkTz6RttpK6tw5LNu8OdTZvn1T/vUAANTP8vHRuqle/mMkPePuHze0bu3T+2PHSo89FtqXXBIC/cQTo/XLyqSDD5YOOyyE+267Sa1aSe++K11wgfTww1tuo3dvadddpSFDpAEDQn+BAQOk4Vv0NshcmzbSvvtKS5ZIS5dGP0T69AmB//HH4axBx47hjMHs2XVfngAAlB4zm+fuFU39XD6e3pe7r1TUg79Jzj5bWrVKuv566aCDwtG2FMJ+yhTpzDOl7bff8nNDhoSzANdfL82bJ+2zj7THHiHst9227m19+KF0001Sly7hh0D16/77Qx1S6Ew4YID0z3/W/OzGjSHI6/rOdGvXhtfgweE7b7pJWrMm/Fj48MPwWrJE+uCDcCbh3/5N+uij8KOhenrUUWH5p5+G1yefRO3q1+LF0ZmGoUOlDh2kTp3CD41335W+8x3puOOiyxgAgMKTl0f6TVH7SL8ub78dTrPHeY29slJq3brmNpcvD6H54INS27ZS377h1adPCPY5c6Tu3cOZhY4dQ5+DJ54IfRPyxdFHSxMnhh8K3buHf8fy5eHHRZ8+0gEHSJ9/HjpIDhgQLo189ln4oVVVFZZXvyorw/cMHSr17x9+tFRWhh8ee+yR9L8UAPJXc4/0SyL0C92770o77hjNd+gQ/Vjo0ye0//AHaaedwo+bXr2knj1DeF5wQThTsd12odPhdttt+ercWXrjDWnDhnCWo2fPcJlj113D2YIHHkju3967d7j8sdNO4WzEEUeEDplt2iRXEwAkjdAvAR98EE65l5fHf5r98celMWPC5YqBA8OPg169wtF+ly7RXRJSmF+1ast2p07hB0jXrmH68suh/8KqVeG7unYNPz6a4pvfDLdlDhoUttWpU/iRsPPO4ZJNZWW4HXPECGnlynBpY9GicNliyRLpooukf//3cLdF67y82AUAWyL0kbjNm0Pnw9atw3TTpnBE7h76MLRtm9n3rF8ffggsWxYuA7iHuyh++tPc1i+FMwvt2klbby39+Mfhh0JlZZh+9ln4cXP++WF+/nxp2LBwCWPVqnCp4vHHpV/8IlyeefddqVu3sH71XRkAkA2EPkrCmjXhSH3JknDXxeLF4bbGRYtCv4D586U335R23z06c9CvX3SG4fXXw+WLm2+W3nsv3to7dw6dKleulP71r3CZ5sMPpW99S5o6NbxfVkZnSQCNI/SBZqiqCn0ZFiyQ/v730CHxrrukXXYJ/SXKy8OPhfLy8EOhoiJc0li+PAR0377hvS5dpBtvDJ0YR4wI3/P4482vq08f6csvw10Tq1eHPh3t2kmnnBJq7t+fHwdAKSP0gTy0cWM4zT9tWribobw8HOF/9FG4BPD97zf/u9u1ky6/PNyG2q1b1koGUAAIfaCAuUtffCHNnBlu37z77tA3YsOG0BHx008b/44RI8Lljo8+Cn0NPv88XNqorAx9Iz76KAxUddll4XJH9yY/qgNAviD0gRKxZk3o1HjzzS3/rsGDQ+fEXr1a/l0A4kPoAyXIXXrqqXBU37171DlxxYro8dLl5dLvfrfl8yhq698/3DHRsWM4I9C+fbjFcYcdpCuuyP2/BUDmCH0AGVu2LNye2BTnnx9uZVy1SjrwQOm736UzIZAUQh9As7zwQnjK5ObN4RbHFSvC4EbbbitdeGHDn33oIWn06DAoEoD4EPoAcmLWLOmee8KlghdekJ5+uu71LrpI2m+/0D+gf/8wxDOA3CD0AcRiw4bwxMdnn214gCOz8KClpUvD5YBBg8KTGkeMCP0MADQfoQ8gEQ8/HEYa3G67MChRJrbbLjwM6rTTwgiJAJqG0AeQNxYsCH0EvvhCWrhQ+s1vQn+Bzz6re/3f/EY644xwxwCAxhH6APLeypXSxReHZw48/PCW7++ySxjqePfdw5DDjDQI1I3QB1Bw/vu/G3964ogR0pFHSuPGhbEHADQ/9MtyUQwAZOInPwkDDK1dG0YYHD16y3XmzJEmTQodAc1Cp8Dp08ODiAA0DaEPIHFbbSVNmBCeTOgeXm+9Jf3oR+GJg+lmzw4jBnbuHH4EPPaYtGlTMnUDhYbQB5CXdtopPK54yZLwI+CNN8IPg622qrne2LFSmzbS+PFhPQD1I/QBFIRddgmXANaulaqqpIkTa97vf8cdUllZeIiQWXjS4Dvv8EMASEfoAyg4ZtI114THBm/aJA0fHr23YEGYvvpqOFtQVhYeOPT++8nUCuQTQh9AQWvVSpo3L4z8949/SHfdFZ4dkO7735cGDAg/Ftq1C69FixIpF0gUoQ+gKGy/vbT33tLJJ0tvvhlO659//pbrbdgQXtV3A9T3LAGgGBH6AIrWr34Vwr+qSnrlFenOO7e813/kyBD+990nrV+fSJlAbAh9AEWvumPf974XhgV2D08PTDduXLgzwEy6++5EygRyjtAHUJIOOiiE/223bfne974XxgfYuDH2soCcIvQBlLTq+/urqsLlgGpLl0pt24Yj/x/8oP6HBQGFhLH3ASCNe3ja34YNW763alUYCRBIGmPvA0AWmElffRXu67/uOmnUqOi9Ll3C6H9m4fbAAj9mQgki9AGgDv36SRdeKD35pHTPPdHy6nH+9903DPxzwQXSunXJ1Ag0FaEPAI048cTwVL/HHtvyvf/5H6ljR2mbbaQvvoi/NqApCH0AyMDWW0uHHBI9BfCvf635/hdfhOA/6SSJbkbIV4Q+ADTDEUdEPwAOPTRafu+90l57hev+dXUGBJJE6ANAC/3tb9JHH2057G+7dtK224aOgUA+IPQBIAt69gz3+a9fL51ySrS8sjLcAmgWlt95Jx3/kBxCHwCyqH176Y47wmA//fvXfO+uu6RTTw0d/8ykyy9n0B/Ei9AHgBwwkxYvDkP5Hndc3etMmSJ17x7W/eSTOKtDqSL0ASCHWreWZsyIOv25S7fcIu2+e831evYM4X/NNQz6g9wh9AEgZmedJb38cgj3a66p+d6kSWHQn9tvT6Y2FDdCHwASNHFiuP4/aVLN5aedJg0fnkhJKGKEPgAkzEy66qpw5L94cbT85ZfDe8uWJVYaigyhDwB5pH//cNtfut69pRtvTKYeFBdCHwDyTPv20pdf1lx23nnhqJ9OfmgJQh8A8lC7diHg33mn5vKyMmny5C1/FACZIPQBII8NGRJO93fvHi276iqpQwdp112TqwuFidAHgDzXvr306afSu+9KXbtGy19/PZzyf//95GpDYSH0AaBADB4srVghLV9ec/mAAeF+/6qqRMpCASH0AaDAdOsWrvffe2+0bNIkqVUr6ZxzkqsL+Y/QB4ACNW5czeCXpKlTpQULkqkH+Y/QB4ACNm5cOOpPv64/eLA0e3ZyNSF/EfoAUAT69ZMefTSaP/DAMJQv9/UjHaEPAEVi7FjplVei+dtvD/f1pw/ti9JG6ANAERk2TFq7tuaygQOlzZuTqQf5hdAHgCKz1VbhtH5FRbSsdWtp332lDRuSqwvJiz30zexWM5tjZpPreb/czB4xs7lmdnPc9QFAsXjpJenqq6P5f/wjDO977LHJ1YRkxRr6ZnaMpFbuPkLSIDMbUsdqp0i6290rJG1tZhV1rAMAyMDFF0sbN0rjx0fL/vjHMJJfesc/lIa4j/RHSpqRas+UtH8d66yQNNTMukjqK2lJPKUBQHFq3Vq67bbwkJ6hQ6Plhx4qTZuWWFlIQNyh31HS0lS7UlKPOtZ5VlJ/SedJ+ldqvRrMbELq9P/c5bXHowQA1KldO+m116SZM6NlZ58tXXFFcjUhXnGH/hpJHVLtTvVs/+eS/sPdr5D0lqTTa6/g7tPcvcLdK7qnP3oKANCoMWOkt9+O5n/+8+RqQbziDv15ik7pD5O0uI51yiXtamatJO0jiaElACDLdtxR+uCDaH7s2ORqQXziDv2HJJ1iZjdIOl7SG2Z2Za11rpE0TdLnkrpKqjWyNAAgG/r2jdqPPSbttFNytSAerePcmLuvNrORksZIus7dP5Y0v9Y6L0r6Zpx1AUCp2rhRatMmtN95J3T627Qp2ZqQO7Hfp+/uK919RirwAQAJat265mh9mzdLQ+q6mRpFgRH5AKDElZWFEfyqw/6998J9/Cg+hD4AQJL0r3/VnDcL1/pRPAh9AIAkqVWrcMRfXh4tGzuWx/MWE0IfAFBDZaX00EPR/JAh0rJlydWD7CH0AQBbOPpo6RvfCO0FC6TevbnOXwwIfQBAnZ5+Who0qOayCROSqQXZQegDAOq03XbhKL+qKlp2yy0151FYCH0AQIPMpIULo/lWraSlS+tfH/mL0AcANGrgQGn/tIeh9+kTntiHwkLoAwAyMnu2NGVKNL/bbtzOV2gIfQBAxn7+c+kXv4jmDz44uVrQdIQ+AKBJLrkkas+aJY0cmVQlaCpCHwDQJGY1T+s//bT0+98nVw8yR+gDAJrlq6+i9hlnMGpfISD0AQDN0ratNH9+NN+7d3K1IDOEPgCg2XbbTfrWt6L5uXOTqwWNI/QBAC3ypz9F7b324ja+fEboAwBapF076d57o/myMumVV5KrB/Uj9AEALTZuXM35PfaQHnggmVpQP0IfAJAV7tKvfx3Nf/e70n33JVcPtkToAwCy5pxzpA8/jObHjZPmzUuuHtRE6AMAsqp3b+nNN6P5igrpk0+SqwcRQh8AkHU77yzNnBnN9+yZXC2IEPoAgJwYM0a69dZonsF7kkfoAwBy5owzovayZdzDnzRCHwCQUytXRu2yMmnhwuRqKXWEPgAgp7p0CZ35qu2wg7RhQ3L1lDJCHwCQcy+9VHOwnnbtkqullBH6AIBYfOc70tlnR/PpT+hDPAh9AEBsbropap93XnJ1lCpCHwAQGzPpJz8J7WeekdauTbaeUkPoAwBiNWVK1O7USdq4MbFSSg6hDwCIVadO0hVXRPM//nFytZQaQh8AELtLLw1D9UrhOv/mzcnWUyoIfQBAItLH5h89Ork6SgmhDwBIRJ8+0qBBof3009KLLyZbTykg9AEAiXn++ai9zz6MzZ9rhD4AIDE9ekh33BHNl5VJixYlV0+xI/QBAIk65RTphBOi+epT/sg+Qh8AkLjp06UxY6L5v/wluVqKGaEPAMgL6b35jzpKWrEiuVqKFaEPAMgbf/pT1OY0f/YR+gCAvHHUUdIRR4T26tXSE08kW0+xIfQBAHkl/Wj/kEOkZ59NrpZiQ+gDAPJKq1bSk09G8wccIH3ySXL1FBNCHwCQd0aNkubNi+Z79kyulmJC6AMA8tLw4dLPfhbNz52bXC3FgtAHAOStX/4yau+1l1RVlVwtxYDQBwDkLTPpmWei+QceSK6WYkDoAwDy2gEHSIMHh/Zxx0kffphsPYWM0AcA5L277oraffsmV0ehI/QBAHlvn32km2+O5l99NblaChmhDwAoCBMmRLfuDRuWbC2FitAHABSM3/42at95Z3J1FCpCHwBQMI4+OmqfempydRQqQh8AUFDSr+enH/mjcbGHvpndamZzzGxyI+tNNbMj46oLAFAYdt1V6t07tH/wg2RrKTSxhr6ZHSOplbuPkDTIzIbUs94Bknq6+1/irA8AUBjSj/Cfeiq5OgpN3Ef6IyXNSLVnStq/9gpm1kbSLZIWm9nRtd8HAODb347aBx+cXB2FJu7Q7yhpaapdKalHHeucKulNSddJ2tvMzq29gplNMLO5ZjZ3+fLlOSsWAJC/qu/bL6N3Wsbi3lVrJHVItTvVs/09JE1z948l3SVpVO0V3H2au1e4e0X37t1zViwAIH8dfniYVlVJK1YkW0uhiDv05yk6pT9M0uI61nlP0qBUu0LS+7kvCwBQaPr0idrdukm3355cLYUi7tB/SNIpZnaDpOMlvWFmV9Za51ZJo8zsGUk/lPRfMdcIACgQP/xh1D7tNIkrvg0zd493g2blksZIeiZ1Cr9FKioqfO7cuS0vDABQkD7/XOrSJbT320967rlk64mDmc1z94qmfi727g/uvtLdZ2Qj8AEA6NxZGjMmtJ9/Xnr99WTryWf0eQQAFLz77ovaPIynfoQ+AKDglZdL//u/oV1VJb31VrL15CtCHwBQFNKH5D3++OTqyGeEPgCgKLRuLU2YENqvvZZsLfmK0AcAFI1z08Zw/f3vk6sjXxH6AICiMXRo1D7jjOTqyFeEPgCgqEyfHrW/+iq5OvIRoQ8AKCrpnfiO5lmtNRD6AICiYiYdemhoP/oonfrSEfoAgKLz8MNRe7fdpJhHnM9bhD4AoOiUlUn33BPNn3RScrXkE0IfAFCUTjwxak+fLj35ZHK15AtCHwBQtFasiNqjR9ObP+uhb2Ydsv2dAAA0R9eu0rvvRvN//nNyteSDRkPfzNqa2QGpdiszO7KRj/zCzK7ISnUAALTQ4MHS3nuHdqmPyZ/JkX5XSY+n2q0lTW9gXUnqJalLS4oCACCbLrooaj/xRHJ1JC2T0P8q9ZK7fyVpU/qbZna3mXVOW9RL0qtZqxAAgBY65pioPX58cnUkLZPQr5K02cxuMbMvJHUys5Vm9oWZjZF0oqTXzWyv1PrDJM3JUb0AADTLlClhunRpzaF6S0lTOvL9P0lHS1or6d8lvZL6/OeSLpX0dzO7UNJ6d38j24UCANASkyZF7fTb+UpJ6wzWOUiSu/vrkmRmm9z9aTP7LPW+u/ttqfmHJN2Qo1oBAGi2Nm2kWbOkkSPD/Lp10lZbJVlR/Bo80jezBxSCPBP7pKbtWlQRAAA5sv/+Ubtjx+TqSEpjp/d/LWmkJJnZCDM7XVJbMztVUt/UOq3N7HeSjpM0WtKxZmY5qhcAgGZr1Uq6+upoPr1dChoMfXd/UtJ8SaYQ/ucrHMn/RNJWkr6U1Cn1/t7u/rSkpZIOzF3JAAA038SJ4VS/JF1yibRyZbL1xCnTjnzu7tdI2kPSOncf5u67KPTSX+fuZ7r76tS6sySNyH6pAAC0nJk0J+0es65dk6slbk0dhre9pPRhdk3SfbXWeV3Sni0pCgCAXNpzT+nCC6P5qqrkaomTeSMPGU4NvLNMUvV4RlWSNkhaJ+mj1HtL3X1Nav1/k9Tb3Wfkquh0FRUVPnfu3Dg2BQAoIu7hEbzp84XCzOa5e0VTP5fJLXubJb0l6WRJ1buktaSOqVdPSW3M7B1JMyX9X1yBDwBAc5lJ48ZFA/UsXy51755sTbnW6Ol9d1/j7nu6+wh33y/12tvdv+nuA9y9vaSBkq6VtJOkf6aO9gEAyGv33BO1S2F43iY/WtfMuptZt/Rl7v6Bu9/m7odKGubuz2WtQgAAcsRMqkidJH/88YbXLQaZPFq3g5n9xIL2kr4v6dT61q8euQ8AgEJwReph8Bs3JltHHDI90r9A0lBJUxXuzd9gZvPM7EMzW1jr9baZTclVwQAAZNOoUVH71SJ/Rmwm1/TXS9qoEPZfKjxad6OkcoXOfR0knZ42fV3ST82sVY5qBgAga9q3j9p//nNydcShsbH3DzOzUQqj8FVI6iFpcPX7qRH41qemX6am/y3pKIVb+wAAyHt77BGml16abB251tiR/p2S7pDUXdJ1kg6RdFJDH3D35939SW9sAAAAAPLE9ddH7cWLEysj5xq8T9/du0mSmS1SCPzzJb1X3+rZLQ0AgHiMHh21Bw4srIF6miKT3vutFH4clElqqzD0bll4yy6TVJ4+rX7lsmgAALLtRz+K2sU6LG8mvffbp15rJb2YardVGHO/h8IlgHKFywDdUsv65aJYAAByJf0U/7XXJldHLjU69r4kmdlxkp5y989S7YXuPs/MzpS0g7tPynWh9WHsfQBAtvTuLS1bFtobNkSP4M03zR17P9P79E+U9J6ZjZM0XdKbZnabpImSnmjqRgEAyEd/+lPUbts2uTpypcGOfGZ2gsK9+bcr3H+/c+qtb0laJekXkjqa2VFpH2ulcIvf/e6+OesVAwCQIxVNPnYuLI09ZW+KpK8Ueua7Qic+k/TH1PsLJK1JLUv/znaS/pp6DwCAgrF+vdShQ2g/8UTNnv2FrsHT++6+s7vvLulASXMkXaQQ/sdKekRSF0n3Sqpw9z1Sr13dfUd3J/ABAAWnfXtp221De1JiPdZyI9Nr+jMUeuW/qXBU/6i7HynpcEknSHrOzKyBzwMAUDAmTw7TF19Mto5syzT0T3P3Y939Q0kD3X2dJLn7XEn7SvpPRuADABSL00+P2puLqHdaRqHv7p+ktd+v9d4md38+24UBAJCUzp2j9p13JldHtmV6pA8AQEkZMCBMzz8/0TKyitAHAKAOl6UGlP/8c+nTT5OtJVsIfQAA6nDyyVG7umNfoSP0AQCoQ9u20tFHh/a6dcnWki2EPgAA9TjiiDC9++5k68gWQh8AgHrsuWfUPvPM5OrIFkIfAIB6DB8uDRkS2tOnJ1tLNhD6AAA04OGHw3TdOqmqKtlaWorQBwCgAdVH+pI0dWpydWQDoQ8AQCPKy8O00AfqIfQBAGjE7beH6ebN0rx5ydbSErGHvpndamZzzKzBoQ7MrIeZvRxXXQAA1OfII6P2aaclVkaLxRr6ZnaMpFbuPkLSIDMb0sDq/yWpQzyVAQDQsAMOCNP0h/EUmriP9EdKmpFqz5S0f10rmdnBktZK+jiesgAAaNiFF4bpc88lW0dLxB36HSUtTbUrJfWovYKZtZV0qaSJ9X2JmU0ws7lmNnf58uU5KRQAgHQjRkTthQuTq6Ml4g79NYpO2XeqZ/sTJU1191X1fYm7T3P3Cnev6N69ew7KBACgpm7dovYOOyRXR0vEHfrzFJ3SHyZpcR3rHCLpHDObJWl3M/tdPKUBANCwK6+M2ps2JVdHc8Ud+g9JOsXMbpB0vKQ3zOzK9BXc/UB3H+nuIyW94u5nxVwjAAB1uvjiqH3IIcnV0Vyxhr67r1bozPeCpFHuPt/d6711LxX8AADkhbIyqWfP0O7XL9lamiP2+/TdfaW7z3B3euYDAArO9deH6Zw5ydbRHIzIBwBAE2yzTZi+957knmwtTUXoAwDQBGPHRu23306ujuYg9AEAaIL27aNb9m68MdlamorQBwCgiQYPDtNCe9QuoQ8AQBOl37p3//3J1dFUhD4AAE104IFR+/jjk6ujqQh9AACayEyaNSuaP/vsxEppEkIfAIBmOOggqWPH0J42LdlaMkXoAwDQTI8/HrXXrUuujkwR+gAANNO++0btJUuSqyNThD4AAC2w/fZh+o1vJFtHJgh9AABa4KwCehYsoQ8AQAtMmRK1589PrIyMEPoAALSAWdR+8MHk6sgEoQ8AQAsdckiY3ndfsnU0htAHAKCFxowJ07feSraOxhD6AAC00PjxUXv9+uTqaAyhDwBAC/XoEbUXLUqujsYQ+gAAZEH1ffqHHZZsHQ0h9AEAyIKddgrTDz5Ito6GEPoAAGTB/fdH7WXLkqujIYQ+AABZ0KaN1KpVaF9ySbK11IfQBwAgS4YNC9MNG5Ktoz6EPgAAWTJhQpjec0+yddSH0AcAIEuOOipqT56cXB31IfQBAMiSXr2iR+1edVWytdSF0AcAIItefDFq59tAPYQ+AABZ1Lt31F64MLk66kLoAwCQZYcfHqaVlcnWURuhDwBAlm29dZgef3yyddRG6AMAkGUDByZdQd0IfQAAsmzSpKi9cWNyddRG6AMAkGXVp/cl6c03k6ujNkIfAIAcqA7+8eOTrSMdoQ8AQA6MGhWm8+dLy5cnW0s1Qh8AgBy47baonS+9+Al9AAByoLxcGj06tGfNkjZtSrQcSYQ+AAA5k360v/POiZXxNUIfAIAc6dMnGp3vvfeSrUUi9AEAyKlrronaa9cmV4dE6AMAkFO77Ra1n3oquTokQh8AgNicfnqy2yf0AQDIsQsvDNPhw5Otg9AHACDHvv3tMJ05M9k6CH0AAHIs/br+WWclVwehDwBAjnXpIvXrF9q33ppcHYQ+AAAxmD076QoIfQAAYtG3b9R+4IFkaiD0AQCIgVnUPvHEZGog9AEAiMlll4Xphg3JbJ/QBwAgJmeeGbXnzYt/+4Q+AAAx6ddPat8+tK+7Lv7tE/oAAMRo/PgwnTFDWrUq3m0T+gAAxOjyy6P2McfEu21CHwCAGPXoIZ13XmjH/dQ9Qh8AgJhdfHHUdo9vu4Q+AAAx69kzaldWxrfd2GtTy84AAAl2SURBVEPfzG41szlmNrme9zub2d/MbKaZPWhmbeOuEQCAuDz2WHzbijX0zewYSa3cfYSkQWY2pI7VTpZ0g7uPlfSxpEPjrBEAgDgMHRqm114b3zbjPtIfKWlGqj1T0v61V3D3qe5e/bunu6RP4ykNAID47LNPmL7ySnzbjDv0O0pammpXSupR34pmNkJSubu/UMd7E8xsrpnNXb58eW4qBQAgh37606gd17C8cYf+GkkdUu1O9W3fzLpKulHSGXW97+7T3L3C3Su6d++ek0IBAMilHXeM2nGNzhd36M9TdEp/mKTFtVdIddy7X9LF7v5+fKUBABCfsjJpzz1D+5FHYtpmPJv52kOSTjGzGyQdL+kNM7uy1jpnShou6RIzm2VmJ8RcIwAAsTj33DB96aV4ttc6ns0E7r7azEZKGiPpOnf/WNL8WuvcJOmmOOsCACAJ1Uf6mzaFQXrMcru92O/Td/eV7j4jFfgAAJSsb34zasfx8B1G5AMAICFm0aN24zjFT+gDAJCgbbYJ07vvzv22CH0AABK0f+qetjvuyP22CH0AABI0aVLU/uc/c7stQh8AgARV9+CXpBdfzO22CH0AABJ22mlhOnVqbrdD6AMAkLDy8jB97TVp48bcbYfQBwAgYRddFLX32y932yH0AQBIWI8e4SVJn+bwgfKEPgAAeeDvfw/TDz7I3TYIfQAA8sAuu0Tt117LzTYIfQAA8kDbtlF7t93CA3iyjdAHACBP/OxnUXvy5Ox/P6EPAECeuPZaadddQ/vqq7P//YQ+AAB5ZObMqP2HP2T3uwl9AADySM+eUv/+oX3uudn9bkIfAIA8c8EFYfrxx9n9XkIfAIA8c/bZUfuLL7L3vYQ+AAB5pn37qH3XXdn7XkIfAIA8NGJEmP7wh9n7TkIfAIA8dMUV2f9OQh8AgDw0enTU/s//zM53EvoAAOQhM+nkk0P7V7/KzrC8hD4AAHlqypSoXZaFxCb0AQDIU4MH1xyPv6WP3SX0AQDIY9deG7WrR+prLkIfAIA8l36vfktG6SP0AQDIcyedFLV79Wr+9xD6AADkOTPpoota/j2EPgAABeCXv5SuuaZl30HoAwBQICZOlFatav7nCX0AAApI587N/yyhDwBAiSD0AQAoEYQ+AAAlgtAHAKBEEPoAAJQIQh8AgBJB6AMAUCIIfQAASgShDwBAiSD0AQAoEYQ+AAAlgtAHAKBEEPoAAJQIQh8AgBJB6AMAUCIIfQAASgShDwBAiSD0AQAoEYQ+AAAlgtAHAKBEEPoAAJQIQh8AgBJB6AMAUCIIfQAASkTsoW9mt5rZHDOb3JJ1AABA08Qa+mZ2jKRW7j5C0iAzG9KcdQAAQNPFfaQ/UtKMVHumpP2buQ4AAGii1jFvr6Okpal2paThzVnHzCZImpCa/crMXs9yndhSN0mfJV1EkWMf5x77OPfYx/HYqTkfijv010jqkGp3Ut1nGhpdx92nSZomSWY2190rsl8q0rGfc499nHvs49xjH8fDzOY253Nxn96fp+h0/TBJi5u5DgAAaKK4j/QfkjTbzLaXdJikcWZ2pbtPbmCdfWOuEQCAohTrkb67r1boqPeCpFHuPr9W4Ne1zueNfO20HJSKLbGfc499nHvs49xjH8ejWfvZ3D3bhQAAgDzEiHwAAJSIggl9RvLLvcb2n5l1NrO/mdlMM3vQzNrGXWMxyPTv1Mx6mNnLcdVVTJqwj6ea2ZFx1VVMMvj/RbmZPWJmc83s5rjrKxap/w/MbmSdjLOvIEKfkfxyL8P9d7KkG9x9rKSPJR0aZ43FoIl/p/+l6PZVZCjTfWxmB0jq6e5/ibXAIpDhPj5F0t2p2/e2NjNu42siMyuXdLvC+DX1rdOk7CuI0Bcj+cVhpBrZf+4+1d0fS812l/RpPKUVlZHK4O/UzA6WtFbhxxWaZqQa2cdm1kbSLZIWm9nR8ZVWNEaq8b/jFZKGmlkXSX0lLYmntKKyWdIJklY3sM5INSH7CiX0a4/S16OZ66B+Ge8/MxshqdzdX4ijsCLT6H5OXTa5VNLEGOsqJpn8LZ8q6U1J10na28zOjam2YpHJPn5WUn9J50n6V2o9NIG7r87gDrYmZV+hhH5WRvJDgzLaf2bWVdKNks6Iqa5ik8l+nihpqruviq2q4pLJPt5D0jR3/1jSXZJGxVRbschkH/9c0n+4+xWS3pJ0eky1lZomZV+hBCMj+eVeo/svdQR6v6SL3f39+EorKpn8nR4i6RwzmyVpdzP7XTylFY1M9vF7kgal2hWS+Htumkz2cbmkXc2slaR9JHF/eG40KfsK4j59M9tG0mxJTyg1kp+k49IH9qljnX0zOC2ClAz38Q8kXS1pfmrRTe5+X9y1FrJM9nOt9We5+8j4Kix8Gf4tby3p/xROhbaRdKy7L63j61CHDPfx3pJ+r3CKf46k77j7mgTKLXjV/x8ws10kndSS7CuI0Je+7sU4RtIzqVNyzVoH9WP/xYP9nHvs49xjH+ePpvy3KJjQBwAALVMo1/QBAEALEfoAAJQIQh+AzKxjqpc1gCJG6AOQwr2+m8zMM3j9X/WHzOzADD+T/uqb4L8TKGmtky4AQF7oJ+krSRtS8+9JukHS1FrrzZL0Udr8xtS0PMNtzE/7DICYEfoA5O5fj4tuZntJ2lbSX2qPCmhmvSR9kLZoU+rzjY4emBqD/evPAIgfp/cB1HaZpGfd/bX0hWbWXuFBSwvSFm+stc5ndZzOf6nW9xP6QEIIfQCSvu7Md4ekgyX9MG1519RT/y5XGEr11Qa+Zp2kUe5u7m6SLpC0PodlA2gCTu8DJc7M+kg6XiGgqyR9q9ZR/mZJjyp09rvK3Rt6pHJVhssAJIDQB0qYmbVTeAZ3mcJjZn/n7jWOzN39czPr6e4rMvnKDJcBSACn94ES5u5fKTyg4xuSjpa0rq7b7CSlX6s/qYGvbC/pqbTP/Sq1DEAe4EgfKHHuvjrVXCfpj5J+2sDqr0r6soH3d9SWR/Z03APyBKEPoFqVpDXuvri+FcysSg1co+dx1kB+4/Q+gJZozoEDw/0CCeFIH0C68WY2vpF10v+/0UaSUtfvM9WmyVUByAqO9AFUc0l3KQypW99rtWp2zGst6fPq+/IbekkamPYZAAkw96b8QAeAiJm1ltQxk2v5ZlYmaRuFHwn8jwdIAKEPAECJ4PQ+AAAlgtAHAKBEEPoAAJQIQh8AgBJB6AMAUCIIfQAASsT/B8Pure3tuXbRAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 576x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.rcParams['font.sans-serif'] = ['SimHei']\n",
    "def plot_precision_vs_recall(precisions, recalls):\n",
    "    plt.plot(recalls, precisions, \"b-\", linewidth=2)\n",
    "    plt.xlabel(\"召回\", fontsize=16)\n",
    "    plt.ylabel(\"精度\", fontsize=16)\n",
    "    plt.axis([0, 1, 0, 1])\n",
    "\n",
    "plt.figure(figsize=(8, 6))\n",
    "plot_precision_vs_recall(precisions, recalls)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 通过选择阀值来实现最佳的精度/召回率权衡"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 目标设定为90%的精度，阀值大概在100000左右 , 设置了阀值为30000\n",
    "y_train_pred_90 = (y_scores > 100000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8915156291042816"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "precision_score(y_train_5, y_train_pred_90)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6260837483859066"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "recall_score(y_train_5, y_train_pred_90)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 总结\n",
    "* 获得了一个90%精度的分类器，但如果召回太低，精度再高，也不怎么有用\n",
    "* 如果工作中，需要99%的精度，你应该回应，召回率是多少？"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ROC曲线"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 本质是 真正类率tpr和假正类率fpr（错误的分为正类的负类实例比例）\n",
    "# 与召回/精度曲线非常相似"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import roc_curve\n",
    "\n",
    "fpr, tpr, thresholds = roc_curve(y_train_5, y_scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAf0AAAF6CAYAAAATeYHoAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nOzdd3xUVeL+8c+ZSQ+BBAhVVECUIgRDpAgCBim2XV0LIGVBUay7lsUKiIiuri7r174oKOoC4v5sa0EEFTtVoyIqCkqTDultZs7vj5kgYiAJJHOTO8/79fLLZHIz85Avmyfn3HPPNdZaRERExP08TgcQERGR8FDpi4iIRAiVvoiISIRQ6YuIiEQIlb6IiEiEUOmLiIhECJW+iIhIhAh76RtjmhpjPqzgmJnGmE+NMRPDlUtERMTtwlr6xpgUYDaQeIhj/gR4rbW9gDbGmHbhyiciIuJm4R7p+4GhQM4hjukPzA89Xgj0qeFMIiIiESEqnG9mrc0BMMYc6rBEYHPo8W4g/cADjDGXA5cDJCYmdmvfvn31BhURkWrntxZrIbj7e/CxP2CxWEp8FmOCnyvxB/AYCB1G2Wbx1lqKSgNEe82+5wLWklvkw2MMXmMo9vmxgMcYArV4m/nYqOCY22Ao8vmpFxsV+jj4f0p8ARJiojAm+FxhUTEFu7ZQUlRIQv0UCnL27LTWplb1fcNa+pWUB8SHHtejnNkIa+0MYAZARkaGXbFiRfjSiYjUMTZUtgFrsYT+DJWvxRKwUFTqZ9OeQgpL/BT5/GzcXcAv2UUkRHv3fU3A/va1NuwuwOsxxEV58VsbPCZg2ZVfEvzaGC9fb84mUMXuLfuhbw74E8o/NxxfznP7S4oNlqfHY/AYg8cEB587cotJiouiRYP4YLmGPucxZt/HWMvPuwvodnQKxhi8nuDnPR7Duh35nNW5GbFRXjweg9dAYWmAYxsl4PUYYqO9NEqMISUxhkaJMcRFe6v2jQjJz8+nTZs2BGKiefaZpxk6dCjGmJ8P57VqY+mvJDil/xmQBnznbBwRkd8rKvXjC1j8gWDR+a0lp7AUXyBYfsHnoaDER16xD4/H7FeYZcVrKfVbNu4pICkuOvg6AcumPYV4PRDl9YTK2e4r6kCoqDfuLmTL3kKKfX5+3JFPiwZx+143sN/X7Ckodfpb9TtNkmLxhgrYH7Dszi+hy1EN2JpTRPrRKUR5DD/vLuCkVsl4PeY3ZWst5Jf4aN04MTi69xgM4PUYOjSvT2yUh2ivh0b1Yoj2eoiN8lQ0u1xrlZaWEh0dTWJiIg8//DAnn3wyrVu3PqLXdLT0jTEdgYuttfuv0n8F+NAY0wI4A+jpSDgRcQ1rg6PPLXsLKfVbtuwtxGMMa7fnkhDjJWtjNk3qx+4r74CFDbsK8AcsX23O5phGCXy3NZeAtSTGRpFb5HP6r/Q7W7KLKjzmwFGs4dePAQpK/Hg9hlPaNsIfsPj8lthoD11bJf/meE9o1GwM7MkvoVXDBOJCo12PCRZwiS9Aw8QYkuKiaZIUS7MGcYc90o1EX331FcOGDWPKlClceOGFXHTRRdXyuo6UvrW2f+jPb4CJB3wuxxjTHxgI/MNamx32gCISVjZUtIHfjGgtOYU+ikr97Movxh8Inv8tG0VvzSkiymPw+S3f/JLDjzvyWLcjn935JRSW+omL9uAvG4kf4and1Vt+XXt8YOEnxUYFp3Y9htyi4Ej/+CZJmFD5eT2G9Tvy6dCiPgkx3t9ML3tM8Jxusc9PsS9A29R6+0bA23KLOC61HrHRnmAx89vC9phgSTdJiqV140Qa1YslruxYU1bOweOS4qLxeurmaDfSWGt54oknuOGGG0hOTqZhw4bV+vq1cXofa+0efl3BLyK1UKk/QLEvgM8fYG9BKSX+ACW+AOt25rMrrxif37Itp4iCUj8rftpNiS9AqT9Y2r6AZUducY3mKyoNlPt8alIsRzdMYFtOERnHpLA1p4i0VskUFPtpk5q4r3Q9xlBY6qdVSjyJsVE0rhdLTJSHhgkxREcZYqO8KlKpVrt37+ayyy7jpZdeYsiQIcyePZsmTZpU63vUytIXkSOTW1RKiS8QHOlay668EgKh88fbcorwGIMvECC/2MeO3GIKSvys3Z5HiS/Aqg17OLphAqu35NAoMebXRVyBX0fg+SX+as/s2W90WjZSLSz1E+01NGsQx/acYtJaJeMNncf1eAw/7cynR+uGRHk95BSV0q5JPY5vmsTxTZNo1iCOaK8hyuPZN7IWqc0WLVrEa6+9xgMPPMD111+Px1P9V9UbW4svaagMrd6XusSGRrmFpX5yi3z4/AG+35ZHYamfUl+AvGIfm/cWkrVxL4WlfhomxgSnvfeb1l66fjfHNanHtuwicot9JMR4g5c92eAlUf4jncuuovpxURSU+PFbS/tm9Sn2+Sko9pPWqgHHNk6kaVIcibFe/AFom5pIq4YJ+6a946K9JMZ4VcgSsfx+P1lZWaSnB69OX7duHW3atKnw64wxK621GVV9P430RSoQCJX0pj2FbM8twue3bNpbiNcYvt6STaPEGHwBS36xj7Xb8mhaP5Zl63fTqmFCcFRc7OebXw61H1XV/bA9b9/jgkOMuhvXiyXKY8guLCXKa/adM964u4CMY1OI8nhIjPXSrH48MVEe6sdH0TQpjvrx0TSIjyYu2kNibNS+a6CN59eFXNHe4CppETk8mzdvZuTIkXz22Wd89913HH300ZUq/COh0peIV+zzs25HPiW+AF9vyabUF2BPQSmPvPcDKQnR7MwrOazXPdhqaq8nWKAl/gBHh0a963fmc05aC2KjPJT6A8RFeUlOiKZji/o0iI/+9dIkA15jiPJ6aJgYQ7TXkJwQg9cYPKFC9oauIRaR2uv1119nzJgxFBYW8vjjj9OqVauwvK9KX1zFWsuOvGI27i4kYC3FpQG+3ZrDxt0FxMdEsXjNNizBKenvtuZWeG56/8KP8Xoo8Qfo0bohSXFRbNxdyMmtU8gu9NG+WRJRHoPfWqI8hhbJ8fgDluYN4vdNZbdIjiO1XqymskUimLWWG264gQcffJCuXbsyb948TjjhhLC9v0pf6oxSf4BVP+/hp135mNCmHoUlfj76YSfrd+azfmf+Eb1+29REGsRHs+aXXM49qSWNEmNo2iCO0zs0oVn9OJW1iBwxYwxer5e//OUv3HfffcTFxYX1/VX6Umvszi9hxgfrsFh25BTzvy+30KphAht2FeA7jMVpR6XE0zI5HmshNtpD/fhoTmqVTIk/QNpRydSLjSI22kObxvWIidK5aRGpGdZann32Wdq1a8cpp5zC/fff79ggQqUvYWetZdWGPSz8Zhsvr9qML7QNZ3nW7fjt6D0pLop6sVHEx3jpfmzDfddJF5b46dmmET3bNOLoRgk1/ncQEamM3NxcrrzySv7zn/8watQoTjnlFEdnDVX6Ui3KLkXz+S35JT4+W7eLD7/ficXiD8DWnEI+/mFXpV4rOSGay05tQ4P4aI5Kiad140Rio7w0rhdDlFaLi0gdsWLFCoYNG8b69eu56667uPXWW52OpNKXqiv2+fn4h50s+W4Hsz89rBs97dO9dUP6HZ9Kt2NS6NyyAYmx+icpInXfsmXL6NOnD82aNWPJkiX06dPH6UiASl8O4dutOaz6eS+rt2Tz5le/0CQpjvwSH5v2FB70a8rucJVXHNyfPLN9E/q2a0yDhGisheYN4klr1YCEGP3TExH3CQQCeDweunXrxsSJE7nmmmuqff/8I6GfvBHM5w+QtSmb77fl8tHanWzaG7xVZ4zXw+a9vy/2A2/ReXzTegzp1Ix+J6TSsXkD4mN0By0RiVyLFi3ihhtuYMGCBbRo0YLJkyc7Hel3VPoRIreolM837OWLjXuZ/s73eAyVvvPYWV2ac3TDBFokx5NxTAqJMVG0ahivS9hERAje937y5Mncd999tG/fnpycHFq0aOF0rHKp9F1sd34Jt730FQtWb/3d5/Yv/HqxURzftB4NE2Ppf0IqxzdNonmDOJrWj9OlbCIih7B+/XqGDx/O0qVLueyyy3jwwQdJSKi9VxCp9F2kxBfgxx15LF6zjbnLNpY7RX/ysSkkxkYxoH0TLsxoRWyURyN2EZHDNHXqVL799lteeOEFLrroIqfjVEh32avDvty0l4cWryW/2M+n6w5+OVyf4xpz17kn0rpxYhjTiYi4U0FBAbt37+aoo45i79697Nmzh9atW4c1g+6yFwHWbsvl7dVbWfnzHt77bsdBjzvp6GQaJcZy8rEpXNqnta5tFxGpJl999RVDhw4lMTGRpUuXkpycTHJystOxKk2lX8tt3F3A0x//xKyP1x/0mPH92jCoYzNSEqJpk1ovjOlERCKDtZYnnniC66+/npSUFB566CE8nro3oFLp10K5RaX87cUs3l69rdzPZxyTQvoxKVzSuzXNGoT3Zg0iIpEmOzubsWPH8vLLLzNkyBBmz55NkyZNnI51WFT6tUResY/pC78vd0QfH+3l7C7N+dvgE2haXyUvIhJOMTEx/PzzzzzwwANcf/31dXKEX0al76CNuwuY8tpqFn+7vdzPpyRE89HNmdqaVkQkzPx+P4888giXXHIJSUlJLF26lKiouv+zuO7/DeqgQMByzdxVvPnV76+fP+PEZkz5QyeN6EVEHLJp0yZGjhzJkiVLiIuLY/z48a4ofFDph5XPH+Dy51by7gEj+8z2TbjnvM46Py8i4rD//e9/jB07lqKiIp555hlGjx7tdKRqpdIPg9Vbshk3ewW/ZBf95vlLerdm8jkdHUolIiL7e/TRR7nmmms46aSTmDt3LieccILTkaqdSr8GvbRqEzfMz/rd86lJsSy6oR8N4qMdSCUiIuU5++yz2bBhA1OnTiU2NtbpODVCO/LVgKJSPxc/+RmrNuz9zfOntmvMIxenq+xFRGoBay2zZ8/mrbfeYu7cuXVqVb525KslCkp8dJz89m+e+/TWTJo3iHcokYiIHCgnJ4crr7ySOXPm0L9/f/Ly8qhfv77TsWpc3fm1pg5477vtvyn8U9s1Zv3fz1Thi4jUIsuXLyc9PZ0XXniBu+66i0WLFkVE4YNG+tXmrte/YeZHv26s85fM47hhkPsWgYiI1GWlpaVcdNFFBAIBlixZQu/evZ2OFFYq/SO0J7+Evv94j9xi377n/ntFLzKObehgKhER2d+OHTtISUkhOjqal19+mWOOOYaUlBSnY4WdpvePgLWWsx/+aF/hJ8Z4WX3nYBW+iEgt8s4779C5c2emTp0KQNeuXSOy8EGlf9istWRMW8TmvYUATDq7I6unDtGWuSIitURpaSm33HILgwcPpmHDhlx44YVOR3KcGuow7Mkv4aS73tn38Y0Dj+fSPq0dTCQiIvtbv349w4cPZ+nSpVx22WU8+OCDJCQkOB3LcSr9Klq4eiuXP7dy38cTBp/A1acd52AiERE50J49e1i/fj3z58/XCH8/mt6vgqc+XPebwh/d6xgVvohILZGfn89//vMfANLT01m/fr0K/wAa6VfSlNdW88wnPwEQ7TX879o+tG8WGdd1iojUdl9++SVDhw7lu+++o2vXrnTq1EnT+eXQSL8SSv2BfYUPsPrOISp8EZFawFrLY489Rvfu3dm7dy/vvPMOnTp1cjpWraXSr8CCr7fS7va39n384z1nEhOlb5uISG0wZswYrr76ajIzM8nKymLAgAFOR6rVNL1/CJ/8uJMrnv/1HP4d53TE6zEOJhIRkf0NGjSItLQ0rrvuujp1wxynqPQPYuPuAi5+cum+jxff2I+2qfUcTCQiIn6/n7vvvpvmzZtz2WWXMWLECKcj1Sn6tagc/oDl3Ec/3vfxN1MHq/BFRBy2adMmBgwYwB133MHy5cudjlMnaaRfjsufXcGu/BIAXrrqFBJi9G0SEXHSa6+9xtixYykuLmb27NmMHj3a6Uh1ktrsAE8s+ZHF324H4Mr+bUk/OjL3ZxYRqS3WrFnDueeeS9euXZk3bx7HH3+805HqLJX+fvwBy71vfQvAycemcPOQ9g4nEhGJXNnZ2TRo0IAOHTrw8ssvM2TIEGJjY52OVafpnP5+7nr9m32PZ1/S3cEkIiKRy1rLM888wzHHHMMnn3wCwB//+EcVfjVQ6Yes25G3bwOesb2P1Xl8EREH5OTkMGLECMaOHctJJ53EMccc43QkV1Hph4x46tfL8yaf3dHBJCIikWn58uWcdNJJzJ8/n2nTprFo0SJatmzpdCxX0XAW+GlnPr9kFwEwa0wGxmgDHhGRcHvnnXfw+XwsWbKE3r17Ox3HlTTSBwY9+AEAjRJjyGzf1OE0IiKRY9u2bXz66acA3HzzzWRlZanwa1DEj/TnLN1AiS8AwD8vSnM4jYhI5Fi4cCGjR48mJiaGH374gZiYGJKTk52O5WoRPdLfW1DCbS9/BUDH5vXpf0IThxOJiLhfaWkpt9xyC4MHD6ZRo0a88cYbxMTEOB0rIkT0SP+u19fse/zfK3s5mEREJDLk5OQwaNAgli5dyuWXX86//vUv3fc+jCJ2pL8rr5j/t2oTAH//U2ddoiciEgZJSUmceOKJzJ8/n3//+98q/DCL2NK/es4qANqkJjK8+9EOpxERca/8/Hyuvvpq1q5dizGGp556igsvvNDpWBEpIoe3uUWlfLZuNwDnpx/lcBoREff68ssvGTp0KN999x1dunShXbt2TkeKaBE50p+7bMO+x1f1b+tgEhERd7LW8uijj9K9e3eys7NZtGgR48ePdzpWxAt76RtjZhpjPjXGTDzI51OMMW8aY1YYY/5dExkeXLQWgOtOb6eNeEREasC///1vrrnmGgYMGEBWVhaZmZlORxLCXPrGmD8BXmttL6CNMaa8eZ5RwH+stRlAkjEmozoz7MkvoaDED8DFPXQuX0SkOhUXFwPw5z//mZkzZ/L666+TmprqcCopE+6Rfn9gfujxQqBPOcfsAk40xiQDrYCN1Rlgyfc7AIiL9tAkKa46X1pEJGL5/X7uvPNOunbtSm5uLvHx8VxyySWaTa1lwl36icDm0OPdQHl73n4EHAP8BVgTOu43jDGXh6b/V+zYsaPSbx4IWK574QsAxvZuXbXkIiJSrk2bNpGZmcmUKVPIyKjWyVmpZuEu/TwgPvS43kHe/w7gCmvtVOBbYOyBB1hrZ1hrM6y1GVWZNlr4zdZ9jy/to9IXETlSr776KmlpaaxcuZLZs2fz3HPPkZSU5HQsOYhwl/5Kfp3STwN+KueYFKCzMcYL9ABsdb35f1cGJxl6tWlE43qx1fWyIiIRKRAIcP/993PMMcewatUqRo8e7XQkqUC4r9N/BfjQGNMCOAMYZoyZZq3dfyX/34GnCU7xfwrMrY43zikqZdGabQD8bfAJ1fGSIiIR6dtvv6VRo0akpqby0ksv0aBBA2JjNZCqC8I60rfW5hBczPcZcJq1NuuAwsdau8xa28laW89aO9Bam1cd7z1/eXA9YEKMl/SjdRcnEZGqstYya9YsunXrxg033ABAkyZNVPh1SNiv07fW7rHWzrfWbq346Orz/Gc/AzCgQ1OtJhURqaLs7GwuvvhiLr30Unr06MF9993ndCQ5DBGxI19uUSk/7SoA4NrM4xxOIyJSt6xevZr09HRefPFFpk2bxjvvvEOLFi2cjiWHISL23v92a+6+x8c31apSEZGqaNKkCU2bNuW5557jlFNOcTqOHIGIGOnP+GAdAKedoF2hREQqY9u2bdx00034fD5SU1P5+OOPVfguEBGl/8P24FrAYxolOpxERKT2W7hwIV26dOHhhx9m1argbci1FsodXF/6Jb4AG3cHz+df1reNw2lERGqvkpISbrrpJgYPHkxqairLly+ne/fuTseSauT6c/rLf9qNL2Bp0ziRlsnxFX+BiEiEGjt2LHPmzGH8+PFMnz6dhIQEpyNJNXN96f+/lZsA6Nm2kcNJRERqJ7/fj9fr5cYbb+RPf/oT559/vtORpIa4uvSttbz0eXDr3e7HNnQ4jYhI7ZKfn89f/vIXYmJiePzxx0lPTyc9Pd3pWFKDXH1O/4uNe/c9Pr1jeTf0ExGJTFlZWWRkZPD000/TsGFDrK2225xILebq0l/+U/CuvPXjoqgX6+pJDRGRSrHW8sgjj9CjRw+ys7NZtGgRd999t1bnRwhXl/7SdcHSv6BbK4eTiIjUDps3b+bWW29lwIABZGVlkZmZ6XQkCSNXD39X/LwHgBbJcQ4nERFx1tdff02nTp046qijWLZsGe3bt9foPgK5eqTvDwTPUXVtpbvqiUhk8vl8TJkyhbS0NJ5//nkAOnTooMKPUK4d6ReU+Cj2+QHo0Ly+w2lERMJv48aNjBgxgg8//JBRo0Zx7rnnOh1JHOba0v9yUzal/uCmPIlaxCciEeaNN95g9OjRFBcX8+yzzzJq1CinI0kt4No2/C50Z70OLTTKF5HI1Lp1a+bOnUu7du2cjiK1hGvP6S/5fgcAJ7Zo4HASEZHwWLNmDU8//TQAZ511FkuXLlXhy2+4tvS/3pwNQPtmSQ4nERGpWdZaZs6cSUZGBrfffjt5ecE7i3q9XoeTSW3jytL3+QNszy0GoJf23BcRF8vOzmb48OGMGzeOnj17smLFCurVq+d0LKmlXHlOf+Oewn2P46L1m66IuFNxcTHdu3fnxx9/5J577uGmm27S6F4OyZWl/9EPOwHocpTO54uI+1hrMcYQGxvL9ddfT5cuXTjllFOcjiV1gCun93/YFly5X+ILOJxERKR6bd26lTPOOIO33noLgCuuuEKFL5XmytLfsLsAgJ5tdD5fRNxj4cKFpKWlsWTJEnbu3Ol0HKmDXFn6od13addUi1lEpO4rKSnhpptuYvDgwaSmprJixQpttiOHxZWlX3aNfueWOqcvInXfq6++yv33388VV1zB8uXL6dSpk9ORpI5y3UK+spvsALRunOhgEhGRI7NhwwaOPvpoLrjgAj7++GOdu5cj5rqR/vqdefseJ8VFO5hEROTw5Ofnc8kll9CpUyfWr1+PMUaFL9XCdSP9rI3Bnfjiol33+4yIRIAvvviCYcOG8f3333PbbbfRqlUrpyOJi7iuGTfvDW7Mc0rbxg4nERGpmkceeYQePXqQk5PDokWLmDZtGlFRrhubiYNcV/prtwen97Uxj4jUNV999RUDBw4kKyuLzMxMp+OIC7n2V0htvysidcGSJUuoX78+J510Eg8//DDR0dEYY5yOJS7lupH+t7/kANCheX2Hk4iIHJzP5+OOO+4gMzOTiRMnAhATE6PClxrlupH+ltA5/UaJMQ4nEREp38aNGxkxYgQffvgho0eP5pFHHnE6kkQI15V+fokfgCZJsQ4nERH5va+//pq+fftSWlrKc889x8iRI52OJBHEVdP71lq8nuDUWP14XaMvIrVP+/btGTp0KKtWrVLhS9i5qvRL/IF9O/JpIZ+I1BZr1qzhjDPOYOfOnURFRfH444/Trl07p2NJBHJV6WcXlgKQnKBRvog4z1rLzJkzycjIYOXKlfz4449OR5II56rS35ZdDOh8vog4Lzs7m+HDhzNu3Dh69epFVlYWPXr0cDqWRDhXlf6OvCIAmtaPcziJiES6CRMm8N///pd77rmHhQsX0rx5c6cjibhr9f6Xm4L77qv0RcQJgUCA7OxsUlJSuPvuuxk7diy9evVyOpbIPq4q/UBoEZ/PH3A4iYhEmq1btzJ69GgKCgp4//33SU1NJTU11elYIr/hsun9EgBObKl990UkfN5++23S0tL2bbbj9erqIamd3FX6ucGFfJreF5FwKCkpYcKECQwZMoQmTZqwYsUKLr/8cm2lK7WWy0pfC/lEJHyKi4t55ZVXuOKKK1i2bBmdOnVyOpLIIbnqnH5WaCFfQ+27LyI16NVXX2XQoEEkJSWxcuVK6tfXDb6kbnDVSL+MSl9EakJeXh5jx47l3HPP5dFHHwVQ4Uud4pqRftn2uwD141zz1xKRWuKLL75g6NChrF27lkmTJnHdddc5HUmkylzTjnsLSvY9jvK6cgJDRBzywgsvMHr0aBo3bszixYs57bTTnI4kclhc045bc4KL+NqmJjqcRETcJj09nfPOO4+srCwVvtRprin97aHL9Ty6VEZEqsGSJUv4y1/+grWWdu3aMW/ePBo3bux0LJEj4prSLyrxA5AQo00xROTw+Xw+7rjjDjIzM1mwYAG7du1yOpJItXFN6e/IC470O7bQSloROTwbN27ktNNOY+rUqYwaNYpVq1ZpdC+u4pqFfDtDW/Cm1tNtdUWk6vx+P6effjpbtmzh+eefZ8SIEU5HEql2rin9nMJSAOrHRzucRETqkqKiIqKjo/F6vcyYMYOWLVty3HHHOR1LpEa4Znr/xx15ACQnaGMeEamcNWvW0L17d+6//34A+vXrp8IXV3NN6e/OD07v14t1zeSFiNQQay1PPfUU3bp1Y+vWraSlpTkdSSQswl76xpiZxphPjTETKzjuMWPMOZV93bJL9ZrU1zl9ETm47Oxshg8fzmWXXcYpp5xCVlYWZ5xxhtOxRMIirKVvjPkT4LXW9gLaGGPaHeS4U4Fm1tr/Vfa194R25EvWOX0ROYRvvvmGV155hXvuuYeFCxfSvHlzpyOJhE24R/r9gfmhxwuBPgceYIyJBp4EfjLG/LGyL5xf7AOggUpfRA4QCAR49913AejVqxc//fQTt956Kx6Pa85wilRKuP/FJwKbQ493A03LOWY08A3wD6C7MebaAw8wxlxujFlhjFmxY8cOAPYUBFfvJ8TonL6I/OqXX35h0KBBDBgwgJUrVwLQrFkzh1OJOCPcpZ8HxIce1zvI+58EzLDWbgWeB3630bW1doa1NsNam5GamkqpP7Dvc3HR+s1dRIIWLFhAWloan3zyCU8++STp6elORxJxVLgbciW/TumnAT+Vc8wPQJvQ4wzg54petCC0BW+92CiM9t4XEWDixImcccYZNGvWjBUrVjBu3Dj9fJCId8Slb4zxhBbeVcYrwChjzHTgImC1MWbaAcfMBE4zxnwAXAU8UNGL7gldrldQ4qt0bhFxt1atWnHVVVexdOlSOnbs6HQckVqhwq+iHJ4AACAASURBVBPgxpgY4EbgXiDOWlsYej4OGEpwYd7bQEJFr2WtzTHG9AcGAv8ITeFnHXBMLnBhVf4Sxb7g9H5MlKb2RSLZnDlz8Hq9DB06lPHjxzsdR6TWqUxLeoAJwLXA5P2efx64DTBAaWXf0Fq7x1o7P1T41aIkVPptU+tV10uKSB2Sl5fHmDFjGDFiBLNnz8Za63QkkVqpMkvdS4B84E1ghTHmU6AdwcvvullrC4wx/pqLWLHcouDvHIlauS8ScT7//HOGDRvG2rVrmTRpEpMnT9a5e5GDqLAlrbUBY0yptfYHY8z1wAbgc2AZ8EdjzPxDv0LN2xU6p98wUfvui0SSdevW0bNnT1JTU3n33Xfp37+/05FEarWqDo23Wmu/MMacBDwEdAQ+rf5YVZNbFFzAl5ygjXlEIoHP5yMqKoo2bdrwf//3f1xwwQW6771IJVR65Zsxpjvw/4wxQwheSrcO2GatXU7wvL5j8oq1MY9IpHj//fc5/vjj+fzzzwG44oorVPgilXTI0jfG9DTGvBr68HPgfoKX3e0muMI+JXT5XbwxZnrovweNMU/UaOoD5IVG+klxKn0Rt/L5fEyePJnMzEyio6O1ha7IYaioJdsQ3Do3GngZmAL8leC19BbIAdoS/OWhdehrvEBcDWQ9qMLS4DrChBhvON9WRMJkw4YNjBgxgo8++ogxY8bw8MMPU6+ertYRqapDlr61dg4wxxiziWDB30ew7AcArxK8Nv9SYK219rwaznpQZTvyxav0RVzp6aefJisri+eff54RI0Y4HUekzqrs/FiJtfZiYA/QACgCLgDqA8cQ/EXAMWWlr3P6Iu5RWFjIN998A8Btt93Gl19+qcIXOUJVPSn2BNAB2EVw6j/DWruy2lNV0d6C4CV78dEa6Yu4wTfffEOPHj0YNGgQhYWFREdHc+yxxzodS6TOq7D0TXCXi1hjTENgHsHz+4kEL9lrUrPxKuenXQUAeD3akEOkLrPWMmPGDDIyMti2bRtPPfUU8fHxFX+hiFRKZUb6sQTP3Q8B5lprv7bWriN43/tnjTFtAUd3xUlNigUgVrfVFamzCgsL9+2Z37t3b7KyshgyZIjTsURcpTIt6QOuITjKv6XsSWvtW8CDQIDgLwaOWb8zH4CUBO3IJ1JXxcbGUlxczL333svbb79Ns2bNnI4k4jqV2YbXB/wn9GH+AZ/7e2j6v1sNZKu0QCC4jjBK0/sidUogEGD69OlcdNFFHH300bzyyivaN1+kBh3xfLgN+rI6whyush8S2oZXpO745ZdfGDRoEBMmTGD27NkAKnyRGlap0jfGxBpjXjLGxIY+bmyMaWKMSTTG+I0xifsd+6wxpndNBS7PzrxiQJfsidQVb731FmlpaXzyySc8+eSTTJw40elIIhGhom14y37tDgB/DP0JMAt4GygluO9+cej4+sAwoEVNhD2Ysmn9erEqfZHabt68eZx55pk0a9aMFStWMG7cOI3wRcKkopH+q8aYP1hrSwGstaXGmMsIruS/0VpbEnza+kLHjya4gc8rNZb4ANaCL2CJ8hhiorR6X6S2sja49uass85i8uTJLF26lI4dOzqcSiSyHLQljTEegjfZmWuMGR16rhXwT+Ama+27BxwfB1wH3FH2S0I4BEI/SHwBRzcFFJFDeP755+nTpw+FhYUkJSVx55136vp7EQcctPSttQFr7R0E76Y3KvT0Q8BSa+2D5XzJ34FfgBnVnvIQVPUitVdeXh5jxoxh1KhReDwecnNznY4kEtEqc8nem8CbxpgAcDOQB8Hz/TY4X2eMMf8EzgV6WmsDB3+16lc2Zdi0vqNbBYjIAT7//HOGDRvGDz/8wOTJk5k0aRJRUVp3I+KkQ/4v0BizACgIfWiBewFPaBX/XmNM99DnzgF6WWu31VjSgwh1PlG6t7ZIrWGt5aqrriI/P593332Xfv36OR1JRKh4pL+K0Mp8giP5DsALBLfd3QJ8AvwfcBQw2Rjz13Cez4dfp/ejvVr9K+K0nTt3EhUVRXJyMnPmzCEpKYnGjRs7HUtEQg45PLbW3matvZPg4j0I3kq3Xuj5R6y1DxPs3a7AycCTNZq2HGUL+Ty65EfEUe+99x5dunThmmuuAaB169YqfJFapjJ32fs7sIhguZ8KjDDGXLP/Mdba7wlex3+GMeYPNRH0oEJD/Q27Cw59nIjUCJ/Px6RJkxgwYAD169fnb3/7m9ORROQgKtqc5wZgHPBXgNDd9UYAfzfGtCk7LPS5LQTP+d9RY2nLUTbS79oqOZxvKyLAxo0b6devH9OmTWPMmDGsXLmSrl27Oh1LRA6iopH+18DZwDIIXrsfuj7/deCBco6fDZxojDmxWlMeQtn1+bqtrkj4eTwetm7dypw5c5g1axaJiYkVf5GIOKaic/oLrbVLCS7cMwTP6UNwRP8HY8wJENybP3T8boIb+pxXY4kPUHYmX9P7IuFRWFjIQw89RCAQoGXLlnz77bcMHz7c6VgiUgmVHR5bgqv0AwDW2iygJ/AzsIRfuxdgLrC4GjMeUtlI/6RWKeF6S5GItXr1arp3785f//pX3n//fQCio3V3S5G6olKlb60tsdZeb63N2e+5FdbaImvtadbaov2e/z9r7Sc1EbY8/lDpF/v84XpLkYhjrWXGjBmcfPLJbN++nQULFpCZmel0LBGpojp/IrzsSr0ob53/q4jUWtdffz3jx4+nd+/eZGVlMXjwYKcjichhqHBPTGNMFNDcWruxEse2Be611l5YHeGqoln9uHC/pUjEuPDCC2nevDkTJkzAo90vReqsymyE3QX4CEgoe8IY0wx4Ezhl/6l9oB7B2+6GjbXBBQXRGumLVBu/3899991HTk4O9957L71796Z3795OxxKRI1SZpiwCDtxatxRIA0oOeL6knGNrlLbhFaleW7ZsYdCgQdx+++38/PPPBAJhvYeWiNSgypS+P/Tf/nwQvP3uAc+H/adDUWkwmtej0hc5Um+++SZpaWl8+umnPPXUU8yZM0fT+SIuUufvc1k2rb8zr7iCI0XkULZv384FF1xAu3btmDdvHh06dHA6kohUszpf+ja0DW+bxvUcTiJSN23bto2mTZvSpEkTFixYQPfu3YmL08JYETeq7LxdA2PMurL/gCzA7P9c6PlFNRe1fPvO6UdpClKkqp5//nmOO+445s6dC0Dfvn1V+CIuVtmRfhFwZyWOawFMOPw4VVdc6icWiNFCPpFKy83N5ZprruHZZ5/l1FNPpU+fPk5HEpEwqGzpF1trZ1d0UGgv/rCWftmmPNmFYb1oQKTOWrVqFcOGDePHH39kypQp3H777URF1fkzfSJSCa75X3rzBvFORxCpE3788UcKCwt577336Nu3r9NxRCSMqlz6xphxwKn8/jI+gAZHnKiKQuv4iNIleyIHtWPHDj777DPOOeccLrzwQs4880zdBlckAlWm9A2/XfCXADQkdK3+ARxbQq/r9EXK99577zFixAjy8/P5+eefSU5OVuGLRKjKlH5c6D8ArLUPAQ+Vd6AxpgMQtjvshfIAKn2RA/l8PqZMmcI999zD8ccfz5tvvklycrLTsUTEQRWWvrX2C/Yr/QrEAGE9uV52yZ5KX+RXpaWlZGZm8tFHH3HJJZfw0EMPaXQvItVza11jTBdjjBf4CmhaHa9ZWb+e09d1+iJloqOjGTJkCHPmzGHmzJkqfBEBKlH6xpgexpiDHhcq+8+BVMALNK++eBUr23vfaKAvEa6wsJCrrrqK999/H4Dbb7+d4cOHOxtKRGqVygyP53KI6X1rrZ/gYr9iYCSwKPSLQFjERntCOcL1jiK1z+rVq+nevTuPP/44S5cudTqOiNRSlVnIVwIUG2OmhD4u7056luAlfNcB/w39IhAWZWWfFOeaLQdEKs1ay5NPPsl1111HUlISCxYsYPDgwU7HEpFaqjJNWVbyfwW+BPoAnwE9gbX8er1+Z6AtkFnNGQ+pbIAfpW14JQK99tprjB8/noEDB/Lss8/SrFkzpyOJSC1WldVvFhhEcCr/T6E/pwNTQ4/PBV6w1u6q7pCHDBUa6pfdYlckEuTm5gJwzjnnMG/ePBYsWKDCF5EKHU5TWn4dYO//3BPAP484URUV+4ITEdqRTyKB3+/n7rvvpm3btmzYsAGPx8PQoUPx6OoVEamEg07vh1bsP0lw972+BFfm7/t0OV+yw1qbU73xKlZW9rpOX9xuy5YtjBw5kvfee4/hw4fToEHYd70WkTruUOf0owneKrce8CbBjXdqrbjosF0wIBJ2b7zxBmPGjKGgoIBZs2YxZswYjK5TFZEqOuicoLW22Fp7BrCBYPFnV/Ba7Y0xF1ZnuKrw6AeguNi8efNo0aIFK1euZOzYsSp8ETkslb3OzR7kz/0NBMYALx5hpirRNrziVmvXriUQCHDCCSfw+OOPExUVRVxcZXfEFhH5vcqu/jGh/5aG/lwUev524N7Q4yeBGGPMGdWasAJl1+mr88VNnnvuOdLT07niiisAqFevngpfRI5YVUb600KPnzngc4bgqv0i4F/AZcBbB3shY8xMoCPwhrV22iGOawossNaeVJmAHrW+uEBubi5XX301zz33HH379uXZZ591OpKIuEhlSj8GiLPWlns5ngmeXPwnwdX9zwJ3GGOirbWl5Rz7J8Brre1ljJlljGlnrV17kPd9gCrcsc+rc5xSx61fv55Bgwaxbt06pkyZwsSJE/F6tUBVRKpPZUr/UX7dda88cQRH+7HW2q3GmMzyCj+kPzA/9Hghwd39flf6xphMIB/YWlG4ss15dE5f6roWLVrQoUMHZs6cSd++fZ2OIyIuVOE5fWvtv6y1xYf4fCHQGtgW+vjzQ7xcIrA59Hg35dyG1xgTA0wCbjnYixhjLjfGrDDGrChbyKfV+1IX7dixg/Hjx5OdnU1sbCyvvfaaCl9Eaky1bONlrf3Z2krd5y6PX6fs6x3k/W8BHrPW7j3E+82w1mZYazPKntOOfFLXvPvuu6SlpfHMM8/w2WefOR1HRCJAuPfuXElwSh8gDfipnGNOB642xrwPdDXGPFWZF9ZCPqkrfD4ft99+O6effjr169dn2bJlujOeiIRFuO9H+wrwoTGmBXAGMMwYM81aO7HsAGvtvrlNY8z71tpxFb2oRvlSl9x0003861//4pJLLuGhhx4iMTHR6UgiEiHCWvrW2hxjTH+CG/n8w1q7Fcg6xPH9K/O6GuVLXVBSUkJMTAw33ngjPXr0YOjQoU5HEpEIE/Zbc1lr91hr54cKv1rocj2pzQoKChg/fjxnn302gUCAli1bqvBFxBGuuB+nLteT2urrr7+me/fuzJgxg/T0dAKBgNORRCSChfucfo1Q50ttY63l3//+N9dffz0NGjRg4cKFDBw40OlYIhLhNNIXqQF5eXncc8899OvXj6ysLBW+iNQKrhjpq/Sltli5ciWdO3cmKSmJjz/+mJYtW+LxuOJ3axFxAVf8NCosOdQuwSI1z+/3c/fdd9OjRw/uv/9+AFq1aqXCF5FaxRUj/XyVvjhoy5YtjBw5kvfee4/hw4dz7bXXOh1JRKRcrij9lsmVvhmfSLV69913GTp0KAUFBcyaNYsxY8ZgdAmpiNRSrih9zaCKU1JTUznuuON4+umnad++vdNxREQOyRV1qTvsSTh9//33/P3vfwegc+fOfPLJJyp8EakTXFH6qnwJl2effZb09HQeeOABtmzZAqDpfBGpM1xR+hrpS03Lzc1l1KhR/PnPf6Zbt25kZWXRokULp2OJiFSJK87pq/OlJllryczMZNWqVUyZMoWJEyfi9XqdjiUiUmWuKH2N9KUmBAIBjDEYY5g0aRLJycn07du34i8UEamlNL0vUo7t27dz9tln88gjjwDwhz/8QYUvInWeK0pfnS/VafHixaSlpfHuu+8SExPjdBwRkWrjktJX68uRKy0t5bbbbmPgwIGkpKSwbNkyxo8f73QsEZFq44rS1/12pDqsWLGCe++9l0svvZTly5fTpUsXpyOJiFQrLeSTiLdmzRo6dOhAr169yMrKonPnzk5HEhGpERrpS8QqKChg/PjxnHjiiSxduhRAhS8iruaKkb5W8klVff311wwbNozVq1dz8803k56e7nQkEZEa54rSX/NLjtMRpA556qmnuPbaa2nQoAELFy5k4MCBTkcSEQkLV0zvpx3VwOkIUodkZ2fTr18/srKyVPgiElFcUfq6ZE8q8vHHH7NgwQIArr/+et58802aNm3qcCoRkfByRelrIZ8cjN/vZ9q0afTr149JkyZhrcXj8eDxuOKfvohIlbjiJ58u2ZPybN68mdNPP51JkyZx0UUXsXjxYs0KiUhEc8VCPpW+HGjz5s2kpaVRWFjI008/zZ///GcVvohEPFeUvn6WSxlrLcYYWrRowTXXXMOwYcNo376907FERGoFV0zvawQnAN9//z19+/ZlzZo1GGOYMmWKCl9EZD+uKH0t5Its1lpmz55Neno633zzDVu2bHE6kohIreSS0lfrR6rc3FxGjRrFmDFjyMjI4Msvv2TAgAFOxxIRqZVcUvpOJxCn/Otf/2Lu3LlMnTqVxYsX07JlS6cjiYjUWi5ZyKfWjySBQICtW7fSokULbr75ZoYMGUL37t2djiUiUutppC91yvbt2zn77LPp3bs3eXl5xMbGqvBFRCrJHSN91PqRYPHixYwcOZI9e/Ywffp0EhMTnY4kIlKnuGKkn1fsczqC1CCfz8dtt93GwIEDSUlJYdmyZVx11VU6rSMiUkWuKP2CEpW+mxlj+OSTTxg3bhzLly+nS5cuTkcSEamTXDG93zIlwekIUgNeeuklTjnlFJo1a8aCBQuIi4tzOpKISJ3mipG+FvK5S0FBAZdffjnnn38+999/P4AKX0SkGrhipK/Od4+vvvqKYcOGsWbNGm655RamTp3qdCQREddwR+lrQZcrLFiwgPPOO48GDRrw9ttvM3DgQKcjiYi4iium91X57tC9e3eGDRtGVlaWCl9EpAa4ovTV+nXXRx99xAUXXEBJSQkNGzbk6aefpmnTpk7HEhFxJVeUvjbnqXv8fj933XUX/fr144svvmDz5s1ORxIRcT1XlL5W79ctmzdv5vTTT2fy5MkMGzaMVatW0bp1a6djiYi4nksW8jmdQKpi+PDhrFq1imeeeYbRo0drIaaISJi4o/Q1vV/rFRcX4/f7SUhI4IknnsDr9XLCCSc4HUtEJKK4YnpfA8Xa7bvvvqNnz55ce+21AHTs2FGFLyLiAJW+1BhrLc888wzdunVj48aNnHfeeU5HEhGJaK4ofV2zV/vk5OQwcuRIxo4dy8knn0xWVhZnn32207FERCKaK0pfq/drn127drFgwQLuuusuFi1aRMuWLZ2OJCIS8dyxkE+lXysEAgFeeeUVzjvvPFq3bs2PP/5IcnKy07FERCTEFSN9rd533vbt2znrrLM4//zzeeONNwBU+CIitYxG+nLEFi1axKhRo9izZw+PPfYYZ511ltORRESkHC4Z6YtT/vGPfzBo0CBSUlJYvnw5V155pTbbERGppdxR+ioZx5x00kmMGzeOFStW0LlzZ6fjiIjIIWh6X6ps/vz5/Pzzz0yYMIGBAwfqNrgiInWEO0b6muAPi/z8fC677DKGDh3Kq6++is/nczqSiIhUQdhL3xgz0xjzqTFm4kE+38AY85YxZqEx5mVjTExFr+kLBKo/qPzGl19+SUZGBjNnzuTWW2/lvffeIyrKFRNFIiIRI6ylb4z5E+C11vYC2hhj2pVz2AhgurV2ELAVGFLR6+7MK67eoPIbe/bsoU+fPuzdu5d33nmHe+65h+joaKdjiYhIFYV7qNYfmB96vBDoA6zd/wBr7WP7fZgKbD/wRYwxlwOXA8Q0O45WKQk1kTXiFRYWEh8fT0pKCrNnz6Z37940adLE6VgiInKYwj29nwhsDj3eDTQ92IHGmF5AirX2swM/Z62dYa3NsNZmBA+ugaQR7qOPPuKEE07g1VdfBeC8885T4YuI1HHhLv08ID70uN7B3t8Y0xB4GLikMi+qhXzVx+/3M3XqVPr160dMTIz2zBcRcZFwl/5KglP6AGnATwceEFq49yJwq7X258q8qC7Zqx6bNm1iwIAB3HHHHQwfPpxVq1aRkZHhdCwREakm4S79V4BRxpjpwEXAamPMtAOOuRRIB243xrxvjBla0Yuq86vHu+++y4oVK5g9ezbPP/889evXdzqSiIhUI2OtDe8bGpMCDAQ+sNZuPdLXi23ezk575n9MGNz+yMNFoOLiYlatWkWvXr2w1vLLL7/QokULp2OJiMghGGNW7lvXVgVhv07fWrvHWju/Ogq/jM7pH57vvvuOnj17MnDgQHbs2IExRoUvIuJirtiRz6POrxJrLc888wzdunVj48aNzJs3j9TUVKdjiYhIDXNF6WslX+X5/X5GjRrF2LFjOfnkk8nKyuLss892OpaIiISBK0pflV95Xq+XJk2acNddd7Fo0SJdkiciEkFcsXm6BvqHFggEmD59Oqeeeio9evRg+vTpTkcSEREHuGSkr9Y/mG3btnHmmWcyYcIE5s6d63QcERFxkEb6LrZw4UJGjx5NdnY2jz/+OOPHj3c6koiIOMgdpe90gFpo8eLFDB48mI4dO7Jo0SJOPPFEpyOJiIjD3DG9r9bfx+/3A9C/f38eeOABli9frsIXERHANaWv1gd44YUX6NixI1u3bsXr9XLjjTeSkKDbDouISJArSj/S5efnM27cOIYNG0bDhg0pLS11OpKIiNRCrij9SB7of/nll2RkZDBr1ixuu+02PvjgA1q1auV0LBERqYVcspAvclv/nnvuITs7m3feeYcBAwY4HUdERGoxd5R+hHX+7t27yc/Pp1WrVjz22GP4/X7tnS8iIhVyxfR+JN1w58MPP6Rr165cfPHFWGtp2LChCl9ERCrFFaUfCdP7fr+fqVOn0r9/f2JjY3nwwQd11YKIiFSJpvfrgO3bt3PRRRexZMkSRo4cyWOPPUZSUpLTsUREpI5xRen7AtbpCDUqMTGRgoICZs+ezejRo52OIyIidZQrpvdzCt13XXpRURHTpk0jPz+fxMREPvvsMxW+iIgcEVeUfpOkWKcjVKtvv/2Wnj17MmnSJF5//XUAPB5X/L9KREQc5IomccuCNmsts2bNolu3bmzevJnXX3+doUOHOh1LRERcwiWl73SC6jFt2jQuvfRSevToQVZWFmeddZbTkURExEVcsZCvrrPWYoxhxIgRxMTE8Le//Q2v1+t0LBERcRl3jPSdDnCYAoEA999/PxdddBHWWtq0acPNN9+swhcRkRrhitKvi7Zt28aZZ57JTTfdRCAQoKioyOlIIiLicu4o/Tp2Un/hwoWkpaWxZMkSnnjiCf773/8SHx/vdCwREXE5V5zTr0uVX1BQwJgxY2jUqBGLFi3ixBNPdDqSiIhECFeUfl2wceNGWrRoQUJCAm+//TZt27YlISHB6VgiIhJBXDG9X9tn91944QVOPPFE7rvvPgA6d+6swhcRkbBzR+nX0gn+/Px8xo0bx7Bhw+jUqRMXX3yx05FERCSCuaL0a6OvvvqKjIwMZs2axW233caSJUs49thjnY4lIiIRzBXn9Gvj9H5hYSEFBQUsWrSIzMxMp+OIiIi4Y6RfWzp/165dzJo1C4Du3buzdu1aFb6IiNQarij92uCDDz6ga9euXHnllfz0008AxMTEOBtKRERkP64ofSen930+H1OmTOG0004jPj6eTz/9VOfuRUSkVnLHOX2HJvittfzhD3/grbfeYtSoUTz66KMkJSU5kkVERKQirih9pxhjuPjiixk+fDijRo1yOo6IiMghuaP0wzjQLyoqYsKECaSnpzN27FhGjhwZvjcXERE5Aq44px8ua9asoUePHjzyyCP88MMPTscRERGpEleM9Gt6oG+t5emnn+baa68lISGBN954gzPPPLOG31VERKR6uWKkb2p4+f6KFSu49NJL6dGjB1lZWSp8ERGpk1wx0q8pO3bsIDU1lZNPPpkFCxZw+umn4/V6nY4lIiJyWNwx0q/m1wsEAvzjH//g2GOPZfny5QAMHjxYhS8iInWaK0b61Tm7v23bNkaPHs3ChQs5//zzOe6446rvxUVERBzkipF+dVm4cCFpaWl88MEHPPHEE7z44oukpKQ4HUtERKRaaKS/n08//ZTGjRuzePFiOnXqVD0vKiIiUku4YqR/JNvwrlu3jo8//hiAiRMnsnz5chW+iIi4kitK/3DNnTuXrl27Mm7cOPx+P16vl/j4eKdjiYiI1AhXlH5Vp/fz8/O55JJLuPjii+ncuTMLFizQynwREXE9V5zTr4odO3Zw6qmn8v333zNx4kTuuOMOoqIi7tsgIiIRKOLarnHjxpx22mk8/vjjnHbaaU7HERERCRuXTO8fen5/165djBgxgnXr1mGMUeGLiEhEckXpH8qSJUtIS0vjxRdf3Le7noiISCRyRemXN873+XxMmTKFzMxMEhIS+Oyzzxg6dGjYs4mIiNQW7ij9clp/+vTp3HnnnYwcOZKVK1eSnp4e/mAiIiK1iOsW8uXl5VGvXj2uvvpq2rZty/nnn+90JBERkVrBHSN9DEVFRVxzzTV0796d/Px8EhMTVfgiIiL7CXvpG2NmGmM+NcZMPJJj9rdx/Vp69OjBo48+ypAhQ3TdvYiISDnCWvrGmD8BXmttL6CNMabd4RyzP39BDtcNG8KWLVt44403mD59OrGxsTXzFxAREanDwj3S7w/MDz1eCPQ5zGP2CRRm075LN7KysjjzzDOrKaaIiIj7hHsePBHYHHq8GyhvSX2FxxhjLgcuD31YnLXso69btmxZzVHlAI2BnU6HcDl9j2uevsc1T9/j8DjhcL4o3KWfB5Tdxq4e5c80VHiMtXYGMAPAGLPCWptRpJX/2QAACMRJREFU/VFlf/o+1zx9j2uevsc1T9/j8DDGrDicrwv39P5Kfp2uTwN+OsxjREREpIrCPdJ/BfjQGNMCOAMYZoyZZq2deIhjeoY5o4iIiCuFdaRvrc0huFDvM+A0a23WAYVf3jHZFbzsjBqIKr+n73PN0/e45ul7XPP0PQ6Pw/o+G2ttdQcRERGRWsgVO/KJiIhIxepM6dfETn7yWxV9/4wxDYwxbxljFhpjXjbGxIQ7oxtU9t+pMaapMebzcOVykyp8jx8zxpwTrlxuUomfFynGmDeNMSuMMf8Odz63CP0c+LCCYyrdfXWi9GtiJz/5rUp+/0YA0621g4CtwJBwZnSDKv47fYBfL1+VSqrs99gYcyrQzFr7v7AGdIFKfo9HAf8JXb6XZIzRZXxVZIxJAWYT3L/mYMdUqfvqROlTAzv5ye/0p4Lvn7X2MWvtO6EPU4Ht4YnmKv2pxL9TY0wmkE/wlyupmv5U8D02xkQDTwI/GWP+GL5ortGfiv8d7wJONMYkA62AjeGJ5ip+YCiQc4hj+lOF7qsrpX/gLn1ND/MYObhKf/+MMb3g/7d3/7FW13Ucx58v4ApapqbzR02jRvNXFLACq60kZfhjFc5aI10ga4YzFXUrYSOtrezHxqLQFpW2VpllhbHSkpKyFZa2KJJmrnQG1mqLkkCIePXH53Pgy9mFey7ce+mc+3psZ9zz/X7O53z4cLjv8/18vp/3h+NsrxuJhvWYAfu5TpssBW4awXb1kk4+y+8CHgM+DkyXdM0Ita1XdNLHPwVeAlwLbKzlYhBs/6uDFWyDin3dEvSHJJNfHFBH/SfphcCngQUj1K5e00k/3wTcbnvLiLWqt3TSx1OBlbb/AnwZmDlCbesVnfTxzcBC2x8Cfg9cMUJtG20GFfu6JTAmk9/wG7D/6hXoN4DFtp8auab1lE4+p+cDV0taC0yR9PmRaVrP6KSPnwBeVn9+NZDP8+B00sfHAZMljQVmAFkfPjwGFfu6Yp2+pBcADwE/pGbyA97eTOzTT5lzOhgWiarDPr4K+Aiwvh76jO27R7qt3ayTfm4rv9b2uSPXwu7X4Wf5aOAOylBoH/A225v6qS760WEfTwfupAzx/xy4xPbWw9Dcrtf6PSDpLOCdhxL7uiLow567GGcBP6lDcgdVJvYv/Tcy0s/DL308/NLH/z8G82/RNUE/IiIiDk23zOlHRETEIUrQj4iIGCUS9CNi2EkaK0mHux0Ro12CfkSPkDRH0uv2c27CcO6VIGm2pBsbz2+V9P1GkQ8Aq+vyrU7qu6wmgYqIITTucDcgIobMUuBHkpZR1kW3LAYmAnMlmZLA42rbnwWQNBV4joHXUY8FJgC/tb2z7dw/gSWSTrT9fmAHsL3WfxHwPspSo/82X1TT4Y4Ddtje3Ti1ANgGvLlRdixwBLDb9o4B2hoR/UjQj+gBkk4DJgNvAaYDM22vlfRFSkBdCCysZddSsni1rKME6WbQPYIS4Js5v8fU46dTk9lIGg8I+AVwMbC8roFvugG4ynZrZ8Yxtp+r594BrAC2S2oF8j7KF5Bdkp5s1NMHHEXZiOjDHXVMROwjQT+iN8wDHrW9qV7ND2TPFbft8e0nJc0HbrE9cYB6PgZc13ZszxeFRlvOk3Rn/fleYE79+a765/22/15fcxdwbC0zBfil7d1197CLKWmgI+IgZE4/ostJGge8m3K13vJgDbjzgAmS1kh6VtIWSsrO/W7VOUi3AMcDfbYFTAKeAR6nBPczge9QphfGUEYP5jZefyRwDvC4pFmS7gFOpuRtXwL8GLhQ0gLgEeCUWkdEHIRc6Ud0vyso8/RNM22vbT2RtJwyfL/D/WTkknQdsM325wbzxs1Ngep2wF+tj2cpm9pspXzB2ADcaHtl2+u3Au+VtBLYCVxKGSl4gPKF4BLb3633HcyzvWow7YuIfeVKP6KL1bn8jwK37+f8BEknUraSnQvMkzRf0ivais4C3tB2bIykYxuPEySd0s97vKoOya8GbrV9A2Xuf7ztP9e6bwZuk7S6tqfdv4EtlPsS3kS5T+By4MWSplH2Zp+UZX8RhyZBP6K7baYE1EfbjreG97cDpwKfAC6jzJMvA17eVn4XjXn+6lTgH43H34D7mgUkvQb4FeUGuym2lzfatUXSWBfLKJuBnEbb7x1JFwIPA+dRRgXupdy9/01gNmVjnImUVQj3SDpqoE6JiP4l6Ed0Mdu7bK/o59TMOsd+JCUofw9YYXsOZRj9kQ6qf8q2Wg/K3fPteQA2AGfbfqvtP7Sdm05jRYDtNfXYnpUDkpYAX6KMRGyk7MZ2NPApyjLAGcDZwDTKFrivpNzxHxEHIXP6ET2qDoW3hsPXABdI+g1l7v7pwdZnexdlRKDpPuCNBxh1372fc5I0Bvga8HXbT9SDMyj7gX8B+KPtRXWL1qdtPyNpCuA6gtA+MhERA0jQj+hNDzZ+finwLUpWPAO3DeH7XFTr3JNcR9LpwK+BTcBq29e3Ctd1+q0lgpMp0xI7JbUn+3ke5QvD/MZrYW+ugNmUO/sjYhAS9CN6Uys5Tx+wy7Yl/YAyV37yUL2J7W3N55JeBHyFMmT/QeBndcRhse3tNZPfzvra9eznd5CkVcCTthcNVVsjInP6Eb1iLHv/P/e1Dtr+D/B8SUuBC4D1wB2STqqb4EyRdCZlyd8xks6QdAZlPXxf63l9nFXLT2p/c0nHS7qecoW/EbjW9mbgtZS5+A2SrpF0zPB1QUQMJFf6Eb1hAnuT1uzJsFc34Lmfckf8NMpd+J8EHqPcFPcw++bdX9dWb/vzvlrfpbX+RZSlgFOB3wHvsf3tVuE6D38ucCUlkc8ySXfbvnyAv0/zS0xEDBH1k6cjInqIpJNs/7Xt2AmttLeHWPfrgfOBVXW4/kBlx1OWDG62/dAAZR8A/mT7ykNtY0TslaAfERExSmT4LCIiYpRI0I+IiBglEvQjIiJGiQT9iIiIUSJBPyIiYpRI0I+IiBgl/gfU4y9z3Bh/QAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 576x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.rcParams['font.sans-serif'] = ['SimHei']\n",
    "def plot_roc_curve(fpr, tpr, label=None):\n",
    "    plt.plot(fpr, tpr, linewidth=2, label=label)\n",
    "    plt.plot([0, 1], [0, 1], 'k--')\n",
    "    plt.axis([0, 1, 0, 1])\n",
    "    plt.xlabel('假正类率', fontsize=16)\n",
    "    plt.ylabel('真正类率', fontsize=16)\n",
    "\n",
    "plt.figure(figsize=(8, 6))\n",
    "plot_roc_curve(fpr, tpr)\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9584511191853252"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 计算曲线下面积AUC，虚线是随机分类0.5到1\n",
    "from sklearn.metrics import roc_auc_score\n",
    "roc_auc_score(y_train_5, y_scores)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* 召回率TPR越高，分类器的假正类FPR就越多\n",
    "* 虚线表示纯随机分类器的ROC曲线，好的分类器应该远离这条线，向左上角\n",
    "* 是使用精度/召回率 PR曲线，还是使用ROC，关键在于 正类非常少或者更关注假正类而不是假负类，选择PR，反之ROC\n",
    "* 例如：前面例子ROC曲线很不错是因为跟负类 非5 相比， 正类 数据5 数量真的很少"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 训练随机森林分类器，比较SGD分类器的ROC曲线和ROC AUC分数"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* 获取训练集中每个实例的分数\n",
    "* RandomForestClassifier 没有descision_function(),但是拥有dict_proda()方法，sklearn中分类器都有这两个中的一个\n",
    "* dict_proda返回一个矩阵，每行一个实例，每列代表一个类别的概率，比如这个图片 70%是5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\86188\\appdata\\local\\programs\\python\\python38\\lib\\site-packages\\sklearn\\ensemble\\weight_boosting.py:29: DeprecationWarning: numpy.core.umath_tests is an internal NumPy module and should not be imported. It will be removed in a future NumPy release.\n",
      "  from numpy.core.umath_tests import inner1d\n"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "forest_clf = RandomForestClassifier(n_estimators=10, random_state=42)\n",
    "y_probas_forest = cross_val_predict(forest_clf, X_train, y_train_5, cv=3,\n",
    "                                    method=\"predict_proba\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1. , 0. ],\n",
       "       [1. , 0. ],\n",
       "       [1. , 0. ],\n",
       "       ...,\n",
       "       [0.1, 0.9],\n",
       "       [0.4, 0.6],\n",
       "       [1. , 0. ]])"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_probas_forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 绘制ROC曲线，需要决策值不是概率，直接使用正类的概率作为决策值：\n",
    "y_scores_forest = y_probas_forest[:, 1] \n",
    "fpr_forest, tpr_forest, thresholds_forest = roc_curve(y_train_5,y_scores_forest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0. , 0. , 0. , ..., 0.9, 0.6, 0. ])"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_scores_forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAf0AAAF6CAYAAAATeYHoAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nOzdd3xUVdrA8d+ZNAKETmiigqIIQiCEroB0VHYFpQsC0kTsLyhSRUUU14aKS5EqoK6KBUWkKkpLIkGkiIpSIh2SEELKzHn/OBMmhIQkkMxN7jzfz2d2Zs7cufeZLM4zpyutNUIIIYSwP4fVAQghhBDCOyTpCyGEED5Ckr4QQgjhIyTpCyGEED5Ckr4QQgjhIyTpCyGEED5Ckr4QQgjhI7ye9JVSlZRSP+RwzFyl1Cal1HhvxSWEEELYnVeTvlKqLLAAKHGZY7oDflrr5kBNpVQtb8UnhBBC2Jm3a/pOoBcQf5lj2gAfuR+vAm4r4JiEEEIIn+DvzYtpreMBlFKXO6wEcNj9+BQQnvkApdQwYBhAiRIlGtWuXTt/A/UROtMDnfF/dRbHXSjPcGRWx+lM57qk3FOQ8RyXXicXx2UKJLtrecp1Nu/PFJu+cNVLzntprFkclykGnemNV/RZszrukvLMnynTXzyPnzXzIt3ZflYhCkh6ttCAQpExfWiXuXf4eY5LSwOtFYGBnuPSUsHlgoAA8PMzZamp4ExT+AeY8vTzJSWBwwHFi3uuf/asQgMhJblw/bNnzTmLF/e8//x5RfJ5KBYMwcU81zmbAP4BULKEidXlVMTHm1hKl/bEGR9nPmfpUqAc5vMmnTOfyd8vhcTTh0g5n0RwSFmSEk6f0FpXzOvf06tJP5fOAsHuxyXJojVCaz0LmAUQERGhIyMjvRddIZGYnMbR+PMcjU9232d6nHCek2dTcLo0WoNLa1zuhCfbLXiobB6LvHMo84M+/V4BDvdzh1KgLn6uMh6P530Oh+d55uMc7m9ch/s4h/s66nLnTT/uQkwZniuyeF/6e7M77uLnF8WkcL+mMvwtMj1P/2yOi9+X1WfN6m+U03HZ/k3c/8IVCn9/T3nyeVNeorjCz2GO2BEDf/yu+Ne/oHiwOW7dWsVve6FDB8Uttc35166Bb75RtLpdcW9381nj4xSjRin8/WDx4vR4FKEVAa04ddrzt2vYULHvN/hxIzRubP6WTzwBM95SvPsuDBwIwcHwySdw333Qvbt5DHDsGFSqBBUqwPEjnn+HlSrBseOw5nu4/XZT1q0brFsHnyyDdu1M2fvvw4MPwhNj4OWXTdlvv0Hv3nDzzbB0qeecPXrA+fOwbBmUcHdOz5gBc+fCZ59BjRqmLCoKVq+Gf/0LbrnFlCUkwOnTEBoKxYpd2X9biYmJ1KxZk2KBASycP49evXqhlPr7Ss5VGJN+FKZJfzMQBuy1NhzvSklzcSzhMsnc/fxsctpVX+uiL61MX2LmS8N8MWX88sv4Jeb5ssnifRe9nt2X32XOk4svsWy//DJ8OTrcX2JZvS/9vA5Hpufu85HhvBcSWabjMp8v83HpX9YqQzwXPc+QQC5KlI7MiTM9pqzfl3NMmROauvD/b+akm+vPkinJ+6q0NIiLM7XDsmU95TEx4HRCgwa4/x3Cli0mWTVrBhXddbQ9e2D5cpOMGjc2ZZGRJsHcfDMMHeo559ixJvlMmwZBQRAbC489Bv/7H+zdCzfdZI57/nl46y2YMAEefdSUffUVdO0Kd94JK1Z4zpn+f53T6YmzfQM4cwb6dIK6N5qyad+aJB5RC25sZcpWnYbPFkLyCXh8iCn75zysWwE33ggnD3mSX5CCpPNw/LB5DaBJOJyLh99/gxbNTFnTJnCmv/m7pifJ2rXh2Wehbl1P3KVKwcyZnhp5uh9/NJ+pZk1P2WefXfr/2+DB5pbRTTdBdPSlx3788aVljzxibhk1amRuGYWEmNuVSE1NJSAggBIlSjBjxgwaN25MjfRfGFfI0qSvlKoD9NVaZxylvxz4QSlVFegCNLMkuHzmcmlOJCZzzJ3Aj7iT97FMj08mpuTqfIH+DiqXKkalUkGElip24XGlUsUu3MqXDCTQz5FlovLlL2nhu7SGzZvhn39MrTG97JFHwN/fJMny5U35tGmwahVMnAht2piyt9+G8ePh4YfhxRdNWXQ0NG0KERGwbZvnWk2bQnKyaS5OT17jx5ua4LffQseOpuz992H6dBNDetI/dgz+8x+T5DIm/XfeMTXH554zSb9sWYh3j5Bau9aT9M+dgxMnIDHR816Hw3PLqGRJ8zfI2ALYt6/5MXL8uKesWzeTrOvX95TdcQc89RR06uQpK1MGPvjANF1fe62nPC7O0wyebvFiLtGvn7llVLeu5++drlgxGDHi0ven/6Ao6n755Rd69+7N5MmT6dGjBz179syX81qS9LXWbdz3u4DxmV6LV0q1AToAr2it47weYB5orYlPSuNogqmFH4k7z7GE5AuPjyaYZH4sIRmnK+d2dT+HomLJoEuSueexeV46OEASt7AVp9P0kfr7e2qfR4+aJBcaamp2AIcPw6+/QtWqcOutpuyTT2DBAlNbfuwxU7ZnD4SFQa1asHOnKVMKWrQwj8+dM03H586ZRLx3L4wa5Un6+/aZJuGWLT1JP71Wf/asJ+7AQJN80+NL16ABpKRcnEybNTPJukIFT9kdd5jPc801nrKGDaFnT+jQ4eJzvvii6SMOCjLPg4PND5WJEy9OxuPGwZNPepqiwdTwnc5L/+4JCZeWvfPOpWXdu3t+KKWrXRteffXisuBg86Mhs8wJX2RNa817773Hk08+SZkyZShXrly+nl/pIt7BW5B9+kkpzgx95Mkcjcv02J3oz6e6cnW+ciUCCQ0xtfHsknn5kkH4OSSZi8IvNdUkUX931eHUKfj7b5ME05th9+yBN980iffJJ01ZbKxpAq1UCbZv95yvZk3Yvx/++MM8XrQIxowx/aGzZsGAAea42bNh2DDTHztnjqnNjhplkvbJk+YezH3t2ubav/3muU6rVibu//zH1M4B3n3XJN4XXvA00W/ebG533umpQSclmeb14OAr758VIjunTp1i6NChfPrpp3Tu3JkFCxYQGhqa5bFKqSitdURer1EY+/QtobVm1a6jLNt6gEOnkzgaf57487nrNy8Z5E9oqSAqhRSjculiFx5XKlWMyqWDCA0xZUH+fgX8KYQwXC5PM+7Zs6bWWrGipzZ56pSp3ZYu7WmaPX/eNOHu2we//+45V9Om5vnWrXDDDaasc2fTRD1njkm+AJ9/bvpIhw83iT4oyDSjv/eeqSmnJ32AIxkGXqXz9zc3l/s3dP/+prbct69JsumqV4f27T19xSVKmAFVM2d6WgjAJPvz5z2jtdN9//2l1x458tKyZs3MLaPg4ItjESI/rV69mi+++IJXX32VJ554Akfmvph8IEkf+On3E7zy7V62HzxzUXmgn8MkcHdtPPRCn/nFfeclg+TPKPIuIcE0X5csCZUrm7L9+02ts1EjM4oYTN9snz5Qrhx8+KHn/XXqwO7dpnab3gI4eDDMm2f6VPv0MUlw3DgzqOuRR8w9mNp0r17mOulJ388PVq4054qOhnD3ZNkzZ8yPhLQMv4Hr1jVJ/48/PGUVK5rm9KAg0wRfs6YZiPbuuyZRp6tUybyeubk3Y208Xb168MsvF5d17mxuGTkcpp89c1l6M7gQhZXT6SQmJobw8HB69uxJREQENTOOQsxnPp2tYg6eYfq3e9n4+wkAKpQMZGSbG2l+Q3kqlSpG2eLSby4Mlwt27TK15oy1v0ceMc3Bzz3nqWVOmWJqwOvWmZrx+fNw221mOs+iRXD//ea4pUtNrXjoUNN8DSbZvvqqaXZOT/opKaZWnv7DIF2Ke8znjz+aUdkZ7dplfgxUqGD6pOfMMYPK0pUvb6ZAZRz05O8Pn35qEuXNN3vKN20y/dJlynjKXnrJDHTLmLjvvtvcMqpaFR566OIyPz9TLoSvO3z4MPfffz+bN29m7969XHvttQWa8MFHk/7vxxJ49dvfWPmraWMMKebPiNY3MKjl9RQP9Mk/iU9LToaDBz0JMDXVDJhyOs3UJzB9uy1bmj7i7dtNYkxMNLVYl8sMpErv216/3pzvp5/g+utNkrvzTpP0o6M9Sb9sWVMbTh84Bqbm3bfvxUm3fHkzijxzH/KaNeY+Yy169myT4DO2CvbsaW4Z1ax56TQkpUzzfmZZjSPKuPCJECLvvvrqKwYOHEhSUhIzZ86kesb/kAuQTw3kO3T6HG+s3sen0YdwaQjydzCw5fU81PoGyhSXbzE7OXPGjNguVcozqvnYMTNSulEjWLjQlK1cCV26mL7jqCiTLFNTTVIrX940QwcFmVp1SIi5Tx/1DWbe9F9/mRp8etJfs8bUshs08AwAi401565U6eJ+ZyGEb9Fa8+STT/LGG2/QoEEDli1bxs0Zf+Xnkgzku4wTZ5N5e+3vLNlygBSnC3+Hok+T6jzarhaVSskQ3KLk8GH45huTyJs0MWU//giDBsEDD5j+azDzpTt2NFO4Vq82ZfHxptk7MdEsnBIW5pkrvWuXSeYlS5rk/eSTJrGfOgVVqpgfARmbx9O99NKlZekrfmUkzdlCCDBrpPj5+fHoo4/y8ssvU8zL00BsnfTjz6cy+/s/mbtxP+dSnCgF/25QlSc73MR15bPd6E9Y5MQJk5hr1jQ15datzeCxhx7yzL1etcr0gQ8ebPq9HQ5T+963z0zBeuwxk7jLlzfN8RlX8KpUyRxTrJinBp7+/ox90+nTuYQQIj9orVm4cCG1atWiRYsWTJ8+3bLxYrZN+ofPJPHvt3/kxFlTPWtXO5T/63Qzt1QplcM7RUHT2gwOO3HCrFGdmmpW9Zoxwyw0kl5bnzTJ9EUfPep57/XXm5HbgYGefusGDcz76tUzCR/MyPONGy++bkjIxdPG0smiIUKIgpKQkMBDDz3EBx98QP/+/WnRooWlA8Rt26c/5n8xfBR5iHrVSjOpax0irs/fVY3EpbQ2i5e4XJ7k+88/pt+7YkWz1CiY2nqnTqYZfNEizyItjRqZ0ebR0WZwmtNp+tlr1rx4BTMhhCgKIiMj6d27N/v37+e5555j7Nix+GVeOOIKXWmffv7P/C8E/jqRyCfRh/FzKGb0aSgJvwAcOWJq5z/84Cn780+zUEpYmKcsMdEsj/rpp56y9u3N/YYNphnf4TC19IQEs9Z3+iBWPz/Tby8JXwhR1GzdupUWLVqQkpLChg0bGD9+fL4l/Kthy6T/5pp9OF2ae8OrcX0F6bu/Wi6XSbwZ1wH38zOrrj30kNkwBEwzebFiF0/nqlTJvJ5xfW6Hw7QKpKZ61k4PCPC0DgghRFHlci8p2ahRI8aPH8/27du57bbbLI7Kw3ZJ//djCSzffpgAP8UjbWtZHU6ht2WLmdudvvTprl1m2VWlPGUOh6mRr17tWR2tYkVTY+/e3cxBB7OjVlKSWSUuXUiIGVmf1fxvIYSwk9WrV9OgQQNiY2Px8/Nj4sSJ+b5hztWyXdJ/ffU+tIaeEdWpXq54zm/wESkpZoW411/3bMWZvq/3sGFmUB2Y9cr/+ss8/vprz/vnzDGrxqXXzMHMcZ8y5dKV4oQQwpekpqYyduxYOnbsSFpaGvHpX7KFkK1G7+/+J54VO/4h0N/BqLY22VT5Kpw+bfrJr73WbDIydKhJ6C1amOluoaFmKdZvvzXloaGmmX3lSrNaXMatPtM3VRFCCOGxf/9++vTpw5YtWxg6dChvvPEGxYsX3gqnrWr6r39nduzo2+RaqpT27a2wVq82y6e++65ZdKZ9e5P4e/QwA+7Sp7t9/LGp+acvdANmL+/rr/esMCeEECJrU6ZMYc+ePXz44YfMmjWrUCd8sNGUvZ2H47h7xkaKBTj4fswdhIb41kp7SpkV5M6dM89PnDD97gAHDly8PrsQQogrd+7cOU6dOsU111zDmTNnOH36NDVq1PBqDD4/ZW/lTrN5Tq+I6rZP+C+9BP36mWVh03+zFS9uBtGlLzlboYLZhjU1VRK+EELkl19++YWIiAi6deuGy+WiTJkyXk/4V8M2Sf/AKVPFrXdNmRyOLFpOnIDRo8389XSHD8OSJWZ6XFKSKduwAfbu9cyBB7OanTTRCyHE1dNaM3PmTBo3bszp06d56aWXcDiKXgotehFnIz3pX2uDEfvx8Z7NXf76y8xxb9HCrG4HZuvV8ePh0CFTwwczMC99PXkhhBD5Jy4ujnvvvZeRI0dyxx13EBMTQ/uMNawixDZJ/9Bpk/Srlyu6A/iOHDGL3pQu7VnpLizM7OHeqpVn8ZoWLcw+79WqWRerEEL4isDAQP7++29effVVVqxYQWhoqNUhXTFbJP1zKWmcOJtCoJ+DSkWoPz8uDoYMgT17zPMyZcxceIAdO8x9QIBZ7GbuXLPQjRBCiILndDp58803SUhIIDg4mC1btvDUU08VySb9jIp29G6HTpuO7Wplg3E4rNu9KK82bYL5801t/q23TB/9oEFmBH5Wu8EJIYQoeIcOHaJdu3Y8/vjjLFmyBAB/mwyQskXSP3DSNO1fU7ZwN+07ndC1Kyxdap537mzWs7/zTs/iNw6HmXonhBDC+7788ksaNGhAZGQk8+fPZ9iwYVaHlK9s8dPl4IX+/MI9iO/LL+Grr8wgvVq1zOC7b76xOiohhBAA77zzDqNGjaJhw4YsXbqUm2++2eqQ8p0tavoHT5nm/cI2cn/RIjPYLn3u/L/+ZebMHzpktpIVQghReNx9992MGTOGTZs22TLhg12SfnpNv2zhSfrnz8OAARAbC1OnmjKHw6yOt2sXBAVZG58QQvg6rTXz58+nV69euFwurrvuOl5++WWCbPwFbY+kf6rwTNdzueDMGTMoT2t4+21T4xdCCFF4xMfHc//99zNo0CCOHTvG2bNnrQ7JK+yV9C2u6R88aObZv/MOREebsocflvn0QghRmGzbto3w8HA+/PBDnn/+eVavXk2pUqWsDssrivxAvjSXJjHFSckgf8oUD7A0Fq3NIjrjx8N110F4uKXhCCGEyCQ1NZWePXvicrnYsGEDLVu2tDokryryNf00pwuAqmWKoZT35+ifPAmPPgppaWbf+pEjYeNGuP9+r4cihBAiG8ePHyctLY2AgAA+++wztm/f7nMJH2yQ9J3ubeZCillTyx8yBGbMgJgY87xXL/DBf0dCCFFofffdd9SrV48pU6YA0KBBA8qWLWtxVNYo8knfZSr6FA/0s+T648aZ+9OnLbm8EEKIbKSmpvLMM8/QqVMnypUrR48ePawOyXJFvk/f5a7plwzy3kc5eRKOHYNbbjH99tu2mYV2hBBCFA779++nT58+bNmyhaFDh/LGG29QvHjhmdZtlaJf03cn/eKB3kn6Bw5AhQpmY5zZs83ce0n4QghRuJw+fZr9+/fz0UcfMWvWLEn4bkU/6btM0i8R5J3mfX9/GDUK3ngDfLRLSAghCqXExEQ++OADAMLDw9m/f7806WdS5JO+0+T8Aq/px8aa+6pVYfBg2LkT7ruvQC8phBAil3bs2EFERAT9+/fn119/BZDafRaKfNL39OkXXE0/NdUssPPQQ5CQAA0bQt26BXY5IYQQuaS15t1336VJkyacOXOG7777jrryBZ2top/0XQXfp79lC5Qvb/rwS5YssMsIIYTIo4EDB/Lwww/Ttm1bYmJiaNeundUhFWo2GL1v7guiT//sWZPkb7sN9u2Db78FC9b/EUIIkY2OHTsSFhbG448/jsNR5OuxBc4GSb9gavr790PNmmZ1vZYtzaC93r3z9RJCCCHyyOl08uKLL1KlShWGDh1Kv379rA6pSCnyP4vSm/fze57+tGlmpH737mZNfSGEENY6dOgQ7dq1Y9KkSWzbts3qcIokG9X087d5/7//hZ49oUQJadIXQgirffHFFwwaNIjk5GQWLFjAgAEDrA6pSCrySd95oU8/fz7KX39BpUoQHAwyHkQIIay3e/du7rnnHho0aMCyZcu46aabrA6pyLJN835+1PSdTqhRA5580iR/IYQQ1omLiwPglltu4bPPPmPTpk2S8K9S0U/6+bj2/saNpin/vfcgNPSqTyeEEOIKaK2ZP38+1113HT/99BMA//73vwkKCrI4sqLPBknf3BfPh6TfujXEx8PKlSALOQkhhPfFx8fTr18/Bg0aRMOGDbnuuuusDslWbJD0TdYPDri65v30EfolS0KnTlcblRBCiLzatm0bDRs25KOPPuKFF15g9erVVKtWzeqwbKXIJ32AYgEO/BxXPsT+99+hfXtYulSm5wkhhFW+++470tLS2LBhA+PGjcPPzzsbqfkSWyT9AL+r+xiHDsGPP8Jzz8n0PCGE8KajR4+yadMmAJ5++mliYmJo2bKlxVHZV5GfsgfgfxW1fIA2baBLFxg5Mn/iEUIIkbNVq1YxYMAAAgMD+f333wkMDKRMmTJWh2Vrtqjp+13hestOJ7hc5vH//gcdOuRjUEIIIbKUmprKM888Q6dOnShfvjwrVqwgMDDQ6rB8gk/X9MeNg8BAGDTIzM8XQghRsOLj4+nYsSNbtmxh2LBhvP7667LvvRfZIulfySC+uDh4+WXz+K67JOkLIYQ3hISEcOutt/LUU0/Ro0cPq8PxOTZp3s970i9dGvbuhcmToWnT/I9JCCGEkZiYyMMPP8y+fftQSjFnzhxJ+BaxRU3/Spv3b7oJJk3K52CEEEJcsGPHDnr16sXevXupX78+tWrVsjokn+aTNf2jRyExsYCCEUIIgdaad955hyZNmhAXF8fq1asZPny41WH5PK8nfaXUXKXUJqXU+GxeL6uU+lopFamU+m9uzpnXpH/DDWblvcWL8/Q2IYQQufTf//6XUaNG0a5dO2JiYmjbtq3VIQm8nPSVUt0BP611c6CmUiqrdp7+wAda6wggRCkVkdN5/f1yn/RdLujY0Txu3z7XbxNCCJELycnJADzwwAPMnTuXr776iooVK1oclUjn7Zp+G+Aj9+NVwG1ZHHMSuFUpVQaoDhzM6aR+eVhGz+GATz81o/crV87124QQQlyG0+nkueeeo0GDBiQkJBAcHMzgwYNRssxpoeLtpF8COOx+fAqolMUxG4HrgEeB3e7jLqKUGuZu/o+EKxu9X6pUnt8ihBAiC4cOHaJt27ZMnjyZiIgcG2eFhbyd9M8Cwe7HJbO5/iRghNZ6CrAHGJT5AK31LK11hLsLAP9crsj3ww/w0kuwevWVhC6EECKzzz//nLCwMKKioliwYAGLFi0iJCTE6rBENryd9KPwNOmHAX9lcUxZoJ5Syg9oCuS4711ua/r/+Q88+ywsW5arw4UQQlyGy+Vi+vTpXHfddURHRzNgwACrQxI58PY8/eXAD0qpqkAXoLdS6gWtdcaR/C8B8zBN/JuApTmdNLcD+Z5+Glq3hjvuyHPcQggh3Pbs2UP58uWpWLEin376KaVLlyYoKMjqsEQueDXpa63jlVJtgA7AK1rrI0BMpmO2AnXzct7c1vSbNzc3IYQQeae1Zt68eTzyyCN0796dRYsWERoaanVYIg+8Pk9fa31aa/2RO+Hni9ysyLd+fX5dTQghfE9cXBx9+/blwQcfpGnTprycvnmJKFJssSKfI4cpIfHxpkl/0iTQOY4QEEIIkdGvv/5KeHg4H3/8MS+88ALfffcdVatWtToscQVskfRz6tPft8/cT5kCMmVUCCHyJjQ0lEqVKvH9998zbtw4/Pz8rA5JXCFbJH2/HKbshYXBwYPw229eCkgIIYq4o0ePMmbMGNLS0qhYsSI//vgjLVq0sDoscZVskfRz6tP394drrgHZ3EkIIXK2atUq6tevz4wZM4iOjgaQlfVswhZJ/3Kj951OcxNCCHF5KSkpjBkzhk6dOlGxYkW2bdtGkyZNrA5L5CN7JP3L/AJdvx7KlIHBg70XjxBCFEWDBg1i+vTpDB8+nK1bt3LrrbdaHZLIZ95enKdAOC5T04+KgrNnITDQiwEJIUQR4nQ68fPz46mnnqJ79+7ce++9VockCogtkv7luprGjIGRI8G926MQQgi3xMREHn30UQIDA5k5cybh4eGEh4dbHZYoQLZo3s9peEnJklC+vFdCEUKIIiEmJoaIiAjmzZtHuXLl0LKIiU+wR9LPJuvHxsLp096NRQghCjOtNW+//TZNmzYlLi6O1atX8+KLL8rofB9hj6SfTV1/+HAoVw5ktUghhDAOHz7M2LFjadeuHTExMbRt29bqkIQX2bpP/+hRcx8Q4L1YhBCiMNq5cyd169blmmuuYevWrdSuXVtq9z7IJjX9rG3dCtHRMl1PCOG70tLSmDx5MmFhYSxevBiAW265RRK+j7JJTT/7f7wNG3oxECGEKEQOHjxIv379+OGHH+jfvz/33HOP1SEJi9ki6QshhLjYihUrGDBgAMnJySxcuJD+/ftbHZIoBGzRvJ/V1rqzZ5u19mUQnxDCV9WoUYOff/5ZEr64wBZJP6vW/c2b4fffvR+LEEJYZffu3cybNw+Au+66iy1btlBLdhoTGdgj6WdR9t//wv79MohPCGF/Wmvmzp1LREQE48aN4+zZswCy7724hD2SfhZZ398frr8eKlb0ejhCCOE1cXFx9OnThyFDhtCsWTMiIyMpWbKk1WGJQsoWA/kyj953OuHAAahRw6KAhBDCC5KTk2nSpAl//PEHU6dOZcyYMVK7F5dlj5p+pucLF0K7dvDcc5aEI4QQBSp9nfygoCCeeOIJvv/+e8aOHSsJX+TIFkk/c9Y/ccL053/6qTXhCCFEQTly5AhdunThm2++AWDEiBG0aNHC4qhEUWGLpJ957f1Bg2DNGvj4Y4sCEkKIArBq1SrCwsLYsGEDJ06csDocUQTZI+lnqulXqABt28JNN1kTjxBC5KeUlBTGjBlDp06dqFixIpGRkTL3XlwReyT9TM/T0iwJQwghCsTnn3/O9OnTGTFiBNu2baNu3bpWhySKKFsk/Ywr8jmdUK8ejBkDLpeFQQkhxFU6cOAAAPfddx8//vgjM2fOJDg42OKoRFFmi6SfsXn/jz9gzx6YPh0ctvh0Qghfk5iYyODBg6lbtxZ4gkEAACAASURBVC779+9HKSWD9US+sMc8/QyPq1SB5cth2zbLwhFCiCu2fft2evfuzW+//cazzz5L9erVrQ5J2Igtkn7Gqn5ICPz73+YmhBBFydtvv81TTz1F+fLlWb16NW3btrU6JGEztmgAz2rtfSGEKGp++eUXOnToQExMjCR8USBsUdPP2Ke/ZAnExsI998CNN1oXkxBC5MaGDRsoVaoUDRs2ZMaMGQQEBFyytLgQ+cUmNX3PfyDvvQejR5sBfUIIUVilpaUxadIk2rZty/jx4wEIDAyUhC8KlO1q+gMHQvPmUKeOZeEIIcRlHTx4kH79+vHDDz8wYMAA3n77batDEj7CHkk/w+PBgy0LQwghcrRz505atWpFamoqixYt4v7777c6JOFDbNG873BIc5gQomioXbs2vXr1Ijo6WhK+8DpbJP10TiesXw/R0VZHIoQQHrt376ZLly6cOHECf39/Zs6cSa1atawOS/ggWyT99D7948fhjjugQwdr4xFCCDD73s+dO5eIiAiioqL4Q0YYC4vZI+m7e/WDg+HRR6FaNYsDEkL4vLi4OPr06cOQIUNo3rw5MTExNG3a1OqwhI+zx0A+d02/dGl480345x9r4xFCiNGjR/O///2PqVOn8vTTT+OQzUBEIWCPpJ/peZUqloQhhPBxLpeLuLg4ypYty4svvsigQYNo3ry51WEJcYE9kr4766elgZ/fxfP2hRDCG44cOcKAAQM4d+4c69evp2LFilSsWNHqsIS4iC3am9L79B9+GAIDYeZMiwMSQviUb7/9lrCwsAuL7fj5+VkdkhBZskfSd9fs4+NNbb9cOWvjEUL4hpSUFEaPHk3nzp0JDQ0lMjKSYcOGyVK6otCyRdJPt3QppKbCvfdaHYkQwhckJyezfPlyRowYwdatW6lbt67VIQlxWbbo08/I33afSAhR2Hz++ed07NiRkJAQoqKiKFWqlNUhCZErtqrpy1Q9IURBOnv2LIMGDeKee+7hnXfeAZCEL4oUWyR9pRROJ7RoAW3bgstldURCCLvZvn07jRo1YsGCBUyYMIHHH3/c6pCEyDPbNIafOgV//WVusgaGECI/ffjhhwwYMIAKFSqwZs0a7rjjDqtDEuKK2CbplyoFW7bAuXNWRyKEsJvw8HC6devG22+/TYUKFawOR4grZos6sQKCgqBJE2jTxupohBB2sGHDBh599FG01tSqVYtly5ZJwhdFni2SvhBC5Je0tDQmTZpE27ZtWblyJSdPnrQ6JCHyjS2SvlLw/fcweTJs2GB1NEKIourgwYPccccdTJkyhf79+xMdHS21e2ErtunTX7MGpkwxI/dbt7Y6GiFEUeN0Omnfvj2xsbEsXryYfv36WR2SEPnOFklfYRL9hAlmyp4QQuTW+fPnCQgIwM/Pj1mzZlGtWjVuvPFGq8MSokDYonkfTLKfMkUG8gkhcm/37t00adKE6dOnA9C6dWtJ+MLWbJP0hRAit7TWzJkzh0aNGnHkyBHCwsKsDkkIr/B60ldKzVVKbVJKjc/huHeVUl1zeU4iIyEqCs6fz584hRD2FBcXR58+fRg6dCgtWrQgJiaGLl26WB2WEF7h1aSvlOoO+GmtmwM1lVK1sjnudqCy1vrL3J67Rw+IiIADB/IpWCGELe3atYvly5czdepUVq1aRZUqVawOSQiv8fZAvjbAR+7Hq4DbgH0ZD1BKBQCzga+VUv/WWn+e00mVgvr1oXx5kNk1QojMXC4X69evp23btjRv3py//vqLypUrWx2WEF7n7eb9EsBh9+NTQKUsjhkA7AJeAZoopR7JfIBSaphSKlIpFZle9vnnEBkJ5coVQNRCiCLrn3/+oWPHjrRr146oqCgASfjCZ3k76Z8Fgt2PS2Zz/YbALK31EWAxcMnOFlrrWVrrCK11BJgpe0IIkdnKlSsJCwvjp59+Yvbs2YSHh1sdkhCW8nbSj8I06QOEAX9lcczvQE334wjg79yc+NdfrzY0IYSdjB8/ni5dulC5cmUiIyMZMmQISkkVQfi2q076SimHe+BdbiwH+iulXgN6Ar8qpV7IdMxc4A6l1PfASODVnE567DjceitIi50QIl316tUZOXIkW7ZsoU6dOlaHI0ShkONAPqVUIPAUMA0oprVOcpcXA3phBuZ9CxTP6Vxa63ilVBugA/CKuwk/JtMxCUCPvHwIZ5r59R4fn5d3CSHsZsmSJfj5+dGrVy+GDx9udThCFDq5qek7gNHAI8DEDOWLgWcxXeqpub2g1vq01vojd8LPF1WqQmoqyGZYQvims2fPMnDgQPr168eCBQvQWlsdkhCFUm6m7KUAicDXQKRSahNQCzP9rpHW+pxSyllwIeZMAf7+5iaE8C0///wzvXv3Zt++fUyYMIGJEydK370Q2cgxTWqtXUqpVK3170qpJ4ADwM/AVuDfSqmPLn8GIYQoGH/++SfNmjWjYsWKrF27ljay+YYQl5XXgXxHtNbbgQrAW8B0oHq+R5VHO3+Brl1h1iyrIxFCeENaWhoANWvW5M0332T79u2S8IXIhVwnfaVUE+ATpVRnzFS6P4GjWuttWDxV/sgR+Oor2L3byiiEEN6wfv16brrpJn7++WcARowYQQVZilOIXLls0ldKNVNKpS+D+zOmZr8cs5peD6Cse/pdsFLqNfftDaXUewUadSYNGsIXX8Dgwd68qhDCm9LS0pg4cSJt27YlICAAh0M2CRUir3Lq06+JWTo3APgMmAw8hplLr4F44AbMj4ca7vf4AcUKINZshVZUdG3qzSsKIbzpwIED9OvXj40bNzJw4EBmzJhByZIlrQ5LiCLnsklfa70EWKKUOoRJ8C9jkn074HPM3PwHgX1a624FHKsQwkfNmzePmJgYFi9eTL9+/awOR4giK7ftYyla677AaaA0cB64DygFXIf5IWCZXbvgv/+FP/6wMgohRH5KSkpi165dADz77LPs2LFDEr4QVymvnWLvAbcAJzFN/xFa66h8jyqPfvgBRoyA6GirIxFC5Iddu3bRtGlTOnbsSFJSEgEBAVx//fVWhyVEkZdj0ldmlYsgpVQ5YBmmf78EZspeaMGGlzt16sCQIXDjjVZHIoS4GlprZs2aRUREBEePHmXOnDkEBwfn/EYhRK7kZg27IEzffWdgqdZ6J4BSagCwUCnVAggsuBBz1roV9P4/KyMQQlytpKQkHnjgAT7++GPat2/PokWLZN97IfJZbpr304BRmFr+M+mFWutvgDcAF+aHgRBCXLGgoCCSk5OZNm0a3377rSR8IQpAbpbhTQM+cD9NzPTaS+7m/0YFEFuunTgJhw+brXX9/KyMRAiRFy6Xi9dee42ePXty7bXXsnz5clk3X4gCdNWrW2hjR34Ec6VemgrXXAOHDlkZhRAiL/755x86duzI6NGjWbBgAYAkfCEKWK6SvlIqSCn1qVIqyP28glIqVClVQinlVEqVyHDsQqVUy4IKOCvnkxTVqoGs1SFE0fDNN98QFhbGTz/9xOzZsxk/frzVIQnhEy7bvK+UUtpsTO0C/u2+B3gfs9FOU8y6+8nu40sBvYEvCyrgrMycCTf6Qfny3ryqEOJKLFu2jD59+lCvXj2WLVtGnTp1rA5JCJ+RU03/c6XUv7TWqQBa61Sl1FDMSP6ntNYpplinuY8fgFnAZ3mBRZwVBeHhXr2iECKPTP0B7rrrLiZOnMiWLVsk4QvhZdkmfaWUA7PJzlL39DyUUtWB/wBjtNZrMx1fDHgcmJT+I0EIIQAWL17MbbfdRlJSEiEhITz33HMy/14IC2Sb9LXWLq31JMxuev3dxW8BW7TWb2TxlpeAfwCv72o/9hm4/XZvX1UIkZOzZ88ycOBA+vfvj8PhICEhweqQhPBpuZmy9zXwtVLKBTwNnIWL+vuVUuo/wD1AM621K/uzFYyjR+H8KW9fVQhxOT///DO9e/fm999/Z+LEiUyYMAF//9ysByaEKCg5DeRbCZxzP9XANMDhHsV/RinVxP1aV6C51vpogUV6GdOmQcdaVlxZCJEVrTUjR44kMTGRtWvX0rp1a6tDEkKQc00/GvfIfExN/hbgQ8yyu7HAT8CbwDXARKXUY1b051eurLjpJm9fVQiR2YkTJ/D396dMmTIsWbKEkJAQKlSoYHVYQgi3y47e11o/q7V+DjN4D8xWuiXd5W9rrWdgWgAaAI2B2QUarRCi0Fq3bh3169dn1KhRANSoUUMSvhCFTG522XsJWI1J7rcD/ZRSozIeo7X+DTOPv4tS6l8FEejlLPkAZszw9lWFEABpaWlMmDCBdu3aUapUKf7v/2T3KyEKq8smfaXUk8AQ4DEArfWfQD/gJaVUzfTD3K/FYvr8JxVYtNlYtQo+/NDbVxVCHDx4kNatW/PCCy8wcOBAoqKiaNCggdVhCSGykVNNfydwN7AVzNx99/z8r4BXszh+AXCrUurWfI0yB337wcMPe/OKQggAh8PBkSNHWLJkCe+//z4lSpTI+U1CCMtcdiCf1noVmLX3MTX6UsAZTI0+Sil1c/rrWutkrfUppdTPQDfMDwav6NwJusuKfEJ4RVJSErNnz2bUqFFUq1aNPXv2EBAQYHVYQohcyO0uexozSt8FoLWOAZoBfwMbcDfxuy0F1uRjjEKIQuLXX3+lSZMmPPbYY6xfvx5AEr4QRUiukr7WOkVr/YTWOj5DWaTW+rzW+g6t9fkM5W9qrX8qiGCz88svsHevN68ohG/RWjNr1iwaN27MsWPHWLlyJW3btrU6LCFEHuW2pl+oTZ8O77xjdRRC2NcTTzzB8OHDadmyJTExMXTq1MnqkIQQVyDHNTGVUv5AFa31wVwcewMwTWvdIz+Cy61bb4VbbvHmFYXwLT169KBKlSqMHj0ah8MWdQUhfJJK3+4y2wOUCgc2aq2LZyirDHwNtMjYtK+UCnMfG1JA8V4iqEotvXTFOrqHX+OtSwphe06nk5dffpn4+HimTZtmdThCiEyUUlFa64i8vi83P9nPA5mX1k0FwoCUTOUpWRwrhChCYmNj6dixI+PGjePvv//G5fL6HlpCiAKSm6TvdN8ySgOz/W6mcku+HXJorBBC5NLXX39NWFgYmzZtYs6cOSxZskSa84WwEVv81/zAA/Cf/+R8nBAie8eOHeO+++6jatWqREVF8eCDD6KUyvmNQogiwzabW8s23UJcmaNHj1KpUiVCQ0NZuXIlTZo0oVixYlaHJYQoALmt6ZdWSv2ZfgNiAJWxzF2+uuBCzd6CBTBqVM7HCSEutnjxYm688UaWLl0KQKtWrSThC2Fjua0fnweey8VxVYHRVx7OlXE4wM/P21cVouhKSEhg1KhRLFy4kNtvv53bbrvN6pCEEF6Q26SfrLVekNNB7rX4vZ70hRC5Fx0dTe/evfnjjz+YPHky48aNw1/6x4TwCbb4L/3VV6HiQyCVFSFy9scff5CUlMS6deto1aqV1eEIIbwoz0lfKTUEuJ1Lp/EBlL7qiK5AzHY4etSKKwtRNBw/fpzNmzfTtWtXevTowZ133inb4Arhg3KT9BUXD/grDpTDPVc/k5L5EVRePfV/0KKFFVcWovBbt24d/fr1IzExkb///psyZcpIwhfCR+Um6Rdz3wDQWr8FvJXVgUqpWwCv7rAH0KABVKni7asKUbilpaUxefJkpk6dyk033cTXX39NmTJlrA5LCGGhHJO+1no7GZJ+DgKB4KuKSAhx1VJTU2nbti0bN25k8ODBvPXWW1K7F0Lkz4p8Sqn6Sik/4BegUn6cMy9WrYLDh719VSEKr4CAADp37sySJUuYO3euJHwhBJCLpK+UaqqUyvY4d7L/GagI+AFeb2hftBD27PH2VYUoXJKSkhg5ciTr168HYNy4cfTp08faoIQQhUpuavpLuUzzvtbaiRnslwzcD6x2/xDwmptrK0JDvXlFIQqXX3/9lSZNmjBz5ky2bNlidThCiEIqNwP5UoBkpdRk9/OsdtLTmCl8jwP/c/8Q8JoJ46FePW9eUYjCQWvN7NmzefzxxwkJCWHlypV06tTJ6rCEEIVUbpJ+epJ/DNgB3AZsBpoB+/DM168H3AC0zecYhRDZ+OKLLxg+fDgdOnRg4cKFVK5c2eqQhBCFWF4G8mmgI6Ypv7v7/jVgivvxPcCHWuuT+R1kTk6ehNRUb19VCOskJCQA0LVrV5YtW8bKlSsl4QshcnQlo/e1+5a57D3Akl3tH3sM9u+34spCeJfT6eTFF1/khhtu4MCBAzgcDnr16oXDkS8TcYQQNpdt8757xP5szOp7rTAj8y+8nMVbjmut4/M3vNwLDLTqykJ4R2xsLPfffz/r1q2jT58+lC5tyarXQogi7HJ9+gGYrXJLAl9jFt4plN57D66/3uoohCg4K1asYODAgZw7d47333+fgQMHolRWv72FECJ72bYJaq2TtdZdgAOYxB+Xw7lqK6V65GdwuVXSkhX/hfCeZcuWUbVqVaKiohg0aJAkfCHEFcntLns6m/uMOgADgY+vMiYhBLBv3z5cLhc333wzM2fOxN/fn2LFcrsithBCXCq3o3+U+7bFfb/aXT4OmOZ+PBsIVEp1ydcIc+HppyEup3YIIYqQRYsWER4ezogRIwAoWbKkJHwhxFXLS03/Bffj+ZleU5hR++eB14GhwDfZnUgpNReoA6zQWr9wmeMqASu11g1zCu7QoZyOEKJoSEhI4OGHH2bRokW0atWKhQsXWh2SEMJGcpP0A4FiWussp+Mp07n4H8zo/oXAJKVUgNb6kpnzSqnugJ/WurlS6n2lVC2t9b5srvsqudyx7+WXpV9fFH379++nY8eO/Pnnn0yePJnx48fj5+fVFa2FEDaXm6T/Dp5V97JSDFPbD9JaH1FKtc0q4bu1AT5yP16FWd3vkqSvlGoLJAJHchEf1auDfDeKoq5q1arccsstzJ07l1atWlkdjhDChnLs09dav661Tr7M60lADeCo+/nPlzldCSB9E9xTZLENr1IqEJgAPJPdSZRSw5RSkUqpyJziF6IwO378OMOHDycuLo6goCC++OILSfhCiAKTL8t4aa3/1lpnNaI/s7N4muxLZnP9Z4B3tdZnLnO9WVrrCK11BMCSJZCrqwtRiKxdu5awsDDmz5/P5s2brQ5HCOEDvL12ZxSmSR8gDPgri2PaAw8rpdYDDZRSc3I66YoVINOWRVGRlpbGuHHjaN++PaVKlWLr1q2yM54QwityO3o/vywHflBKVQW6AL2VUi9orcenH6C1vtC2qZRar7UektNJ+/QpkFiFKBBjxozh9ddfZ/Dgwbz11luUKFHC6pCEED5C5a5VPh8vqFRZzEI+32utczVQ73KCqtTSH32znn83qHb1wQlRgFJSUggMDOTw4cNs3LiRXr16WR2SEKKIUkpFpXdx54XXt+bSWp/WWn+UHwlfiKLg3LlzDB8+nLvvvhuXy0W1atUk4QshLGGL/Tj3ZTfTXwiL7dy5kyZNmjBr1izCw8NxuVxWhySE8GG2SPqvvGx1BEJcTGvNe++9R+PGjTlx4gSrVq1i2rRp+Pt7exiNEEJ42CLp33ST1REIcbGzZ88ydepUWrduTUxMDB06dLA6JCGE8Pro/QIxeozVEQhhREVFUa9ePUJCQvjxxx+pVq0aDoctflsLIWxAvo2EyAdOp5MXX3yRpk2bMn36dACqV68uCV8IUajYoqYvhJViY2O5//77WbduHX369OGRRx6xOiQhhMiSLaohY0ZbHYHwVelL6W7ZsoX333+fDz74gFKlSlkdlhBCZMkWNf3kFKsjEL6qYsWK3HjjjcybN4/atWtbHY4QQlyWLWr6MmVPeNNvv/3GSy+9BEC9evX46aefJOELIYoEWyT9oGJWRyB8xcKFCwkPD+fVV18lNjYWACW7PQkhighbJH0hClpCQgL9+/fngQceoFGjRsTExFC1alWrwxJCiDyxRdL/4AOrIxB2prWmbdu2LFmyhMmTJ7N27VquueYaq8MSQog8s8VAvpjt0rwq8p/L5UIphVKKCRMmUKZMGVq1apXzG4UQopCyRU2/b1+rIxB2c+zYMe6++27efvttAP71r39JwhdCFHm2SPoNw62OQNjJmjVrCAsLY+3atQQGBlodjhBC5BtbJH0h8kNqairPPvssHTp0oGzZsmzdupXhw4dbHZYQQuQbWyT9mO1WRyDsIDIykmnTpvHggw+ybds26tevb3VIQgiRr2yR9D/6yOoIRFG2e/duAJo3b05MTAyzZ8+mRIkSFkclhBD5zxZJXypk4kqcO3eO4cOHc+utt7JlyxbArLAnhBB2ZYspe716Wx2BKGp27txJ7969+fXXX3n66acJD5fRoEII+7NF0hciL+bMmcMjjzxC6dKlWbVqFR06dLA6JCGE8ApbNO8nn7c6AlGUxMXF0bp1a2JiYiThCyF8itJaWx3DVQmqUkvXa76ByE9lHXSRvR9//JGEhAQ6d+6My+UCwOGwxW9eIYQPUkpFaa0j8vo+W3zrBQVZHYEorJxOJy+88AKtW7dmwoQJaK1xOByS8IUQPskW33zPPGN1BKIwOnz4MO3bt2fChAn07NmTNWvWyDa4QgifJgP5hC0dPnyYsLAwkpKSmDdvHg888IAkfCGEz5OkL2xFa41SiqpVqzJq1Ch69+5N7dq1rQ5LCCEKBVs07y9ebHUEojD47bffaNWqFbt370YpxeTJkyXhCyFEBrZI+ocOWR2BsJLWmgULFhAeHs6uXbuIjY21OiQhhCiUbJH0+/WzOgJhlYSEBPr378/AgQOJiIhgx44dtGvXzuqwhBCiULJF0q9e3eoIhFVef/11li5dypQpU1izZg3VqlWzOiQhhCi0ZCCfKHJcLhdHjhyhatWqPP3003Tu3JkmTZpYHZYQQhR6tqjpR0ZaHYHwlmPHjnH33XfTsmVLzp49S1BQkCR8IYTIJVsk/e+/tzoC4Q1r1qwhLCyMtWvXMnr0aNnzXggh8sgWSb9xnlcfFkVJWloazz77LB06dKBs2bJs3bqVkSNHymI7QgiRR7ZI+q1aWx2BKEhKKX766SeGDBnCtm3bqF+/vtUhCSFEkSQD+USh9emnn9KiRQsqV67MypUrKVasmNUhCSFEkWaLmv7Jk1ZHIPLTuXPnGDZsGPfeey/Tp08HkIQvhBD5wBZJf8kSqyMQ+eWXX36hcePGzJkzh2eeeYZp06ZZHZIQQtiGLZr3y5e3OgKRH1auXEm3bt0oXbo03377LR06dLA6JCGEsBVb1PT79rU6ApEfmjRpQu/evYmJiZGEL4QQBcAWSV8UXRs3buS+++4jJSWFcuXKMW/ePCpVqmR1WEIIYUuS9IUlnE4nzz//PK1bt2b79u0cPnzY6pCEEML2bJH0P1hsdQQiLw4fPkz79u2ZOHEivXv3Jjo6mho1algdlhBC2J4tBvLFJ1gdgciLPn36EB0dzfz58xkwYICsrCeEEF5ii6QvA/kKv+TkZJxOJ8WLF+e9997Dz8+Pm2++2eqwhBDCp9iieb90aasjEJezd+9emjVrxiOPPAJAnTp1JOELIYQFbJH0ReGktWb+/Pk0atSIgwcP0q1bN6tDEkIIn2aLpP/TT1ZHIDKLj4/n/vvvZ9CgQTRu3JiYmBjuvvtuq8MSQgifZoukv2uX1RGIzE6ePMnKlSt5/vnnWb16NdWqVbM6JCGE8Hm2GMjXrJnVEQgAl8vF8uXL6datGzVq1OCPP/6gTJkyVoclhBDCzRY1/VtvtToCcezYMe666y7uvfdeVqxYASAJXwghChlb1PSFtVavXk3//v05ffo07777LnfddZfVIQkhhMiCLWr6sbFWR+C7XnnlFTp27EjZsmXZtm0bDz30kCy2I4QQhZQtkv6qVVZH4LsaNmzIkCFDiIyMpF69elaHI4QQ4jJs0bxftarULL3po48+4u+//2b06NF06NBBtsEVQogiwhY1/U4drY7ANyQmJjJ06FB69erF559/TlpamtUhCSGEyAOvJ32l1Fyl1Cal1PhsXi+tlPpGKbVKKfWZUirQ2zGKS+3YsYOIiAjmzp3L2LFjWbduHf7+tmgoEkIIn+HVpK+U6g74aa2bAzWVUrWyOKwf8JrWuiNwBOjszRjFpU6fPs1tt93GmTNn+O6775g6dSoBAQFWhyWEECKPvF3TbwN85H68Crgt8wFa63e11t+5n1YEjmU+Rik1TCkVqZSKBFiypGCC9XVJSUkAlC1blgULFhATE0O7du0sjkoIIcSV8nbSLwEcdj8+BVTK7kClVHOgrNZ6c+bXtNaztNYRWusI87wgQvVtGzdu5Oabb+bzzz8HoFu3boSGhloclRBCiKvh7aR/Fgh2Py6Z3fWVUuWAGcDg3Jy0d598iU0ATqeTKVOm0Lp1awIDA2XNfCGEsBFvJ/0oPE36YcBfmQ9wD9z7GBirtf47Nyf1s8UcBOsdOnSIdu3aMWnSJPr06UN0dDQRERFWhyWEECKfeDtdLgf6K6VeA3oCvyqlXsh0zINAODBOKbVeKdXLyzH6rLVr1xIZGcmCBQtYvHgxpUqVsjokIYQQ+UhpL3eIK6XKAh2A77XWR672fEFVaumnJn7P1IeqXH1wPig5OZno6GiaN2+O1pp//vmHqlWrWh2WEEKIy1BKRaWPa8sLrzeMa61Pa60/yo+En+7Qofw6k2/Zu3cvzZo1o0OHDhw/fhyllCR8IYSwMVv0hrdoaXUERYvWmvnz59OoUSMOHjzIsmXLqFixotVhCSGEKGC2SPrVr7E6gqLD6XTSv39/Bg0aROPGjYmJieHuu++2OiwhhBBeIOuo+hg/Pz9CQ0N5/vnnGTt2LH5+flaHJEShEx8fz7Fjx0hNTbU6FOFjAgICCA0NLbCB1LZI+n/9BdS3OorCy+Vy8dprr3H77bfTtGlTXnvtNatDEqLQio+P5+jRo1SrVo3g4GCUkl08hXdorUlKSuLwYbOGID6tagAAG8ZJREFUXUEkfls070dGWh1B4XX06FHuvPNORo8ezdKlS60OR4hC79ixY1SrVo3ixYtLwhdepZSiePHiVKtWjWPHLlmBPl/YoqZ/3fVWR1A4rVq1igEDBhAXF8fMmTMZPny41SEJUeilpqYSHByc84FCFJDg4OAC61qyRdJvLIvGXWLNmjV06tSJOnXqsHr1am699VarQxKiyJAavrBSQf77s0XzvvBwOp0AtGnThldffZVt27ZJwhdCCAHYJOknp1gdQeHw4YcfUqdOHY4cOYKfnx9PPfUUxYsXtzosIYTFTpw4Qd++fSlbtiyhoaFMmDDhwmvnz59nxIgRlC5dmkqVKjF16tQLr02ePBmlFA6Hg9DQUHr27MnevXut+Agin9gi6X/ztdURWCsxMZEhQ4bQu3dvypUrJ9OMhBAX6dWrF7GxsXzyySeMHTuWl156iQ8//BCARx99lBUrVrB48WKmTJnCc889xyeffHLhvVWqVGHz5s288cYb7NixgxYtWnDgwAGrPoq4WlrrIn0LrHyjfnharPZVMTExunbt2loppZ999lmdkpJidUhCFGm7du2yOoR8tX//fg3o6OjoC2XdunXTd955p46NjdV+fn566dKlF1574IEHdOvWrbXWWk+aNElfd911F177559/dEhIiB4xYoS3wvdZOf07BCL1FeRMWwzk69LF6gisM3XqVOLi4vjuu+9o166d1eEIIQqZU6dOAaaJP90rr7xCXFwca9aswel00qFDhwuvNWzYkK+/zrr5tHLlynTt2jXb10XhZ4vmfV9z6tQpDh48CMC7775LTEyMJHwhRJbq1q1L9erVGThwIJ9++ilaa2688UYaNWrEnj17CAkJoXz58heOf+CBB1i3bl2256tfvz4HDhwgKSnJG+GLfGaLpO9Ls2t++OEHGjRoQN++fdFaU65cOdksRwiRraCgIL788kuCgoK49957iYiIYNOmTYCp/Wde9a1MmTLUrVs32/OVLVsWgDNnzhRc0KLA2CLpb9xodQQFz+l0MmXKFNq0aUNQUBBvvPGGzCUWwouUurSC0bWrKfvyS0/ZrFmmbNgwT1lsrCnLvHN1o0amPCrKUzZ5simbPNlTlvH1KxEWFsaePXt49913iY2NpU2bNqxYsYLU1FQcDpMGNm/ejFLqwi078r1TtNki6Z8+bXUEBevYsWO0a9eOSZMm0bdvX6Kjo2nUqJHVYQkhipDAwEAeeughfvnlF2655RaGDx9OiRIlSExMBEyz/c8//8zs2bMve57T7i/c0qVLF3jMIv/ZIum3bGl1BAWrRIkSnDt3jgULFrBo0SJCQkKsDkkIn6O1uWX05ZemrGtXT9mwYaZs1ixPWdWqpiw29uL3R0WZ8oy/4SdPNmUZa/pX8xt/9uzZdO7c+cLzChUqMGHCBA4fPkz58uU5deoUcXFxFC9enAYNGlCpUqXLnm/nzp1cf/31sgZIEWWLpF+unNUR5L/z58/zwgsvkJiYSIkSJdi8eTMDBgywOiwhRBFTrFgx1qxZc1Ef/MmTJwkODqZ79+4AfJmhfyImJibbcx0/fpwvvviCe+65p+ACFgXKFlP27GbPnj307t2bmJgYatWqRa9evS70uwkhRF507dqV/2/v3sOjqs8Ejn/fhFwIsAQCJIQqaILIQiSNAgJWE5Vy8REpIiihEgEDVnDBFaI0mADaXayldJWKWA1b27KusCsqC3KpBGlBAw0ga6xQo7BRuVsEkkDIu3+cyTgJwUxCZoZM3s/zzONcfuec9/wc8s753U67du0YPXo0jz/+OIcPHyYnJ4fMzEySkpK45557mDZtGgChoaEX3Hr77NmzFBQU8Le//Y2nnnqKNm3aMGfOnECcimkEQZFJ9u8PdASNQ1V55ZVXuP766ykpKeHtt99m7NixgQ7LGNOERUdHs3HjRiorKxk1ahRPPPEE999/PwsXLgRg+fLl3HPPPTz00EPk5uby8MMPV9v+yy+/pH///syYMYN+/frx/vvv24yhJky0ZidVExPRubtmztzCc7M7BzqUS7ZgwQKefPJJ0tLS+N3vfkd8zaG+xhifKyoqomfPnoEOwzRzdX0PRWSnqtb7HrNB0byfkBDoCC6NqiIipKenEx4ezmOPPUZoaGigwzLGGBNkgqJ5/5prAh1Bw1RWVvLzn/+cMWPGoKpcffXVZGVlWcI3xhjjE0GR9JuiQ4cOMXz4cGbPnk1lZSVlZWWBDskYY0yQC4qkf/JkoCOon/Xr19OnTx/y8/NZunQpK1eupGXLloEOyxhjTJALiqS/fXugI/DemTNnyMjIICYmhoKCAqZMmWLLWhpjjPGLoBjI1xQWqDt48CDx8fFERUXxzjvvkJCQYCtaGWOM8auguNIfMCDQEXy31157jd69e7vnxSYlJVnCN8YY43dBkfQvV6dPn2by5Mnce++99OrVi3HjxgU6JGOMMc2YJX0f+fDDD7nhhht45ZVXmDNnDvn5+XTr1i3QYRljjGnGgiLpv/deoCO4UGlpKWfOnGHjxo08/fTThIWFBTokY0wzs3z5ckQEESEkJISuXbvy2GOPuW+n66tj+usC57PPPnOfX83H8uXL/RJDUxMUA/nKywMdgePYsWOsXr2aiRMn0q9fP/bt20d4eHigwzLGNHMFBQWcPXuWDz74gLlz53Lo0CFeffXVQIfVaJYuXcr1Ne4/fNVVVwUomgvt2rWLzZs3M2PGjECHEhxJ/6abAh0BbNmyhfT0dA4fPsytt95Kt27dLOEbYy4LN9zgLNE+cOBATp8+zfz58/nNb35DREREgCNrHD169HCf4+Vo165dLF68+LJI+kHRvB8ZGbhjV1RUkJubS1paGi1btmTbtm3Wd2+MuWylpKRw9uxZjh07FuhQTAAERdIPFFVlxIgRzJs3j/T0dHbu3ElKSkqgwzLGmIs6dOgQIkJMTAwAJSUljBw5krZt2xIXF8fMmTOprKwEvu0z37VrF6NHj6Z169Zce+21bNu2zb2/jz76iEGDBhEZGcmAAQMoLi6udrwTJ04wfvx4WrduTVxcHPPmzaPq7q6pqalMmTKFvn370r59e9asWcOAAQOIjo7mjTfeaJTzLS8vZ/r06bRv35527doxffp0yj36hDdv3oyIcP78eRYsWEC3bt2qdX2cO3eOrKwsYmNjiYmJISMjg5Mey8CePHmSCRMm0LFjR6Kjoxk1ahRHjhwBIDc3FxHhgQce4PPPP3ePN8jNzW2Uc2sQVW3Sj/C4RP3lyi80UF599VX97W9/G7DjG2Ma10cffRToEBpNXl6eOn/mHXv37tUePXro7bff7n7vlltu0d69e+vGjRt15cqV2r59e83Ly1NV1eLiYgW0d+/eOm3aNN2wYYOmpKRocnKyqqqeO3dOu3fvrgMGDNB169bp/PnzNSwsTLt27ere/w9/+EO95pprdPXq1bp06VJt3bq1/uxnP3Mfu02bNrpq1Srt06ePtmjRQvPy8nTIkCE6bNiwOs+vKr533333omUefPBBjYuL0z/84Q+6YsUKjY2N1czMTPfn7777rgKamZmpffv21V/+8pdaVFTk/jwrK0tjY2P1tdde07ffflsTEhJ07Nix7s+nT5+u8fHxunr1an3zzTc1KSlJJ0+erKqqJSUlWlBQoDk5Odq5c2ctKCjQgoICLSkpqfPc6voeAju0ATkzKPr0S0r8d6yysjJmzZpFSkoKDzzwAOPHj/ffwY0xAdHt8TWBDgGAz/71jgZt57nUd0pKCi+//DLgXPSNGzeOQYMG0atXLyoqKliyZAnvv/8+GRkZ7m169uzJc889B8CcOXO49957Aec+Ivv27WPt2rUkJCQwZMgQCgsL+ctf/gLA1q1bWb9+PYWFhSQnJwPOUuRz587l0UcfBeC+++5j1KhRrF69mtjYWDIyMiguLiY/P9/r80tLS6v2uri4mG7dunHgwAFefvllVq1axciRIwGIiIhg9OjRZGdnc8UVV7i3KSoqYuvWrdXGYpWWlrJ48WJefPFFxowZA8DRo0d58MEHKSsrIzIykgMHDtCnTx9GjBgBQPfu3Tl+/DgA8fHxxMfHs3fvXsLDwy+LcQdB0bzvr1vrFhUV0b9/f55//nn279/vn4MaY8wlKiwsZM2aNYgIs2fP5sorrwScHwNjxozhnXfe4Y477iA2NpbNmzdTWlpabfvMzEz385iYGCoqKgDYt28f7du3JyEhwf35zTff7H6+a9cu2rZt60744DTpnz592v03tHPnzu5YPJ/Xx0svvURhYaH7ER8fD8CePXuorKwkNTW12vErKyvZs2dPtX384he/uGDw9f79+ykvLycjI8PdNJ+RkcG5c+c4cOAAAJMmTWLTpk0MHDiQ2bNnU1JSwsCBA+sVvz8FxZX+97r4dv+qSl5eHtOnTycqKoo1a9YwfPhw3x7UGHPZaOgV9uUiOTmZ5ORkRowYwcKFCxk7diwA33zzDSkpKXTq1Ilx48Yxd+5cXnjhhQu2v9j0t8rKSkJCql87hoaGVntdM4FXvVZXv35jSExMrPbDoibPGC52/L59+16wXVWZ119/ncTExGqfVf1wuvPOO/nrX//KunXryM/PZ9iwYfzkJz9h8eLFDTsZHwuKK31f27FjB5MmTaJ///7s3r3bEr4xpkmaM2cOhYWFbNiwAYBNmzZRXFzM2rVreeSRR7jxxhtrbcWsmcirJCQkcOzYMfdVL8Cf/vQn9/Pk5GS+/vrralfV+fn5REVF0b1798Y6rYu67rrrCAkJqdZVkJ+fT0hICNddd12d2ycmJhIeHk5ZWZn7h1OrVq149tlnOXHiBADPPPMMBw8eZOrUqaxYsYL58+eTl5dXbT+RkZEXtJ4ESlBc6R877pv9HjlyhI4dO9K3b1/WrVvH7bffftEvvzHGXO769evHbbfdxsKFCxk8eLB7BH9eXh5JSUksWbKEP//5z14vbDN06FC6du3Kj3/8Y7Kzs9m5cyerVq2iSxen+fWmm25i8ODBjB07lmeeeYavvvqKJ598kuzsbL+sEXDllVcyadIkpk6dSmlpKarKo48+yuTJk91X6t8lKiqKmTNnMmvWLFSVLl26kJuby4kTJ4iLiwPg448/ZsWKFTz99NO0bNmSN99884Jp2ykpKRw9epRly5bRq1cvtm7dSlZWli9OuW4NGf13OT3C4xJ19nONO3r//PnzunDhQo2KitIPPvigUfdtjLm8BfPofVXVP/7xjwpoQUGBqqr+9Kc/1ZiYGI2NjdWMjAydMmWKJiYmakVFhXt0fHFxsXv7qtHuVYqKijQ1NVWjoqI0JSVFs7Kyqo3eP378uI4bN05btWqlnTp10pycHD1//ryqOqP3c3JyVFV1woQJOmHCBFVVzcnJ0VtuuaXO8/Nm9H5ZWZlOmzZNo6OjNTo6WqdNm6ZlZWUXPZ+azp49q7NmzdKOHTtqmzZt9K677tLPP/+82vlNmDBBO3XqpFFRUXrzzTfrnj17LtjPsmXL9Hvf+562aNFCe/fuXee5+Wr0vmgj9qsEQkTn7jr/2S1kpXdulP0dOnSI+++/n/Xr13P33Xfz0ksv0a5du0bZtzHm8ldUVETPnj0DHYZp5ur6HorITlWt93SAoOjT96Jrxivr16+nT58+bNmyhaVLl/L6669bwjfGGBM0gqJPv7Fs27aNDh06sGnTJnr16hXocIwxxphGFRRX+pfi008/dY82zc7OpqCgwBK+McaYoBQUSf+9rQ3bbsWKFSQnJzN58mTOnz9PaGgoLVu2bNzgjDHGmMtEUCT9+jp9+jQTJ05k3LhxJCUlsW7dOpuKZ4xxa+oDnE3T5svvX1D06Q8a5H3ZI0eO8IMf/IBPPvmE7OxscnJyaNEiKKrBGNMIWrRoQUVFBWFhYYEOxTRTFRUVPstLQZHtQuqxTHOHDh1IS0vjhRdeuOAmDcYYExkZyalTp2zmjgmYb775hsjISJ/su1k07x87doz09HQ+/fRTRMQSvjHmojp27MiRI0c4c+aMNfMbv1JVzpw5w9GjR+nYsaNPjhEUV/pFH8OwpNo/y8/PJz09ncOHDzNixAiuvvpq/wZnjGlSIiMjiY2N5auvvqK8vDzQ4ZhmJiIigtjYWJ9d6QdF0j9ey9r7FRUVPPXUUyxYsICEhAS2b99OSkqK/4MzxjQ5bdu2pW3btoEOw5hGFxTN+9f2uPC9RYsWMW/ePMaPH8/OnTst4RtjjGn2guJK33WjKABOnTpF69atefjhh0lISODuu+8OXGDGGGPMZSQorvQBysrKmDZtGv369eP06dO0atXKEr4xxhjjwe9JX0ReFpFtIpJ9KWU8FRbso3///ixZsoShQ4favHtjjDGmFn5N+iIyCghV1QHA1SLSvSFlPJ0/c5Lch4byxRdfsGbNGhYtWkRERIRvTsAYY4xpwvx9pZ8K/Kfr+XrgpgaWcass/TtX9bie3bt3M3z48EYK0xhjjAk+/m4HbwWUuJ4fB2obUl9nGRHJBDJdL8s/+XDr3i5dujRyqKaGDsDRQAcR5KyOfc/q2Pesjv2jlnlrdfN30j8FVN3GrjW1tzTUWUZVlwHLAERkh6re0PihGk9Wz75ndex7Vse+Z3XsHyKyoyHb+bt5fyffNtf3AT5rYBljjDHG1JO/r/TfAN4TkXhgGHCviDylqtnfUeZGP8dojDHGBCW/Xumr6kmcgXrbgTRV3V0j4ddW5u917HaZD0I1F7J69j2rY9+zOvY9q2P/aFA9i91FyhhjjGkegmZFPmOMMcZ8tyaT9H2xkp+prq76E5G2IrJWRNaLyH+LSLi/YwwG3n5PRSRWRAr9FVcwqUcd/1pE7vRXXMHEi78X7UTkf0Rkh4i86O/4goXr78B7dZTxOvc1iaTvi5X8THVe1l86sEhVfwh8BQz1Z4zBoJ7f02f5dvqq8ZK3dSwiPwDiVPUtvwYYBLys4x8Dv3dN32sjIjaNr55EpB3w7zjr11ysTL1yX5NI+vhgJT9zgVTqqD9V/bWqbnC97Agc9k9oQSUVL76nInIrcBrnx5Wpn1TqqGMRCQNeAj4Tkbv8F1rQSKXu7/ExoLeIRANXAAf9E1pQOQ+MBU5+R5lU6pH7mkrSr7lKX2wDy5iL87r+RGQA0E5Vt/sjsCBTZz27uk3mAo/7Ma5g4s13+X7gI+AZoJ+ITPdTbMHCmzreCnQFHgGKXOVMPajqSS9msNUr9zWVpN8oK/mZ7+RV/YlIe+A5YKKf4go23tTz48CvVfVrv0UVXLyp4+8Dy1T1K+B3QJqfYgsW3tRxDjBVVecDHwMP+Cm25qZeua+pJEZbyc/36qw/1xXo68ATqvq5/0ILKt58T28HHhaRzUCyiPzGP6EFDW/qeD9wtev5DYB9n+vHmzpuBySJSCjQH7D54b5Rr9zXJObpi8g/AO8Bm3Ct5Afc47mwTy1lbvSiWcS4eFnHDwE/A3a73npBVV/zd6xNmTf1XKP8ZlVN9V+ETZ+X3+U2wCs4TaFhwGhVLalld6YWXtZxPyAPp4l/G/AjVT0VgHCbvKq/AyLyj8C4S8l9TSLpg3sU42Bgi6tJrkFlzMVZ/fmH1bPvWR37ntXx5aM+/y+aTNI3xhhjzKVpKn36xhhjjLlElvSNMcaYZsKSvjHG50QkVEQk0HEY09xZ0jcmSIjISBEZeJHPIn15rwQRGSIi/+zx+l9E5B2PIk8Cb7mmb3mzv3TXIlDGmEbUItABGGMazVzgjyKyCGdedJUngG7AfSKiOAt4PKyqLwKIyPeBMuqeRx0KRAIfqurZGp/9HZgjIp1UNQsoB0pd+x8OzMaZanTecyPXcrgtgHJVrfT4aCJwBrjTo2woEA5Uqmp5HbEaY2phSd+YICAiVwJJwAigH5CmqptFZDlOQp0KTHWV3YyzileV7ThJ2jPphuMkeM81v0Nc7/fAtZiNiEQAAnwA3AH8yjUH3tOjwEOqWnVnxhBVLXN9NhZ4HigVkapEHobzA6RCRD7z2E8YEIVzI6KnvaoYY0w1lvSNCQ4TgJ2qWuK6mq+L+4pbVSNqfigiGUCuqnarYz8LgX+q8Z77h4JHLLeJSJ7r+WpgpOv5Ctd/16nqUdc2K4BoV5lkoEBVK113D7sDZxloY0wDWJ++MU2ciLQAJuNcrVd515VwJwCRIrJRRL4Rka9xluy86K066ykXiAHCVFWAROBL4BOc5N4TeBOneyEEp/XgPo/tWwI3Ap+IyGARWQnE4azbPgfIB4aJyERgB9DZtQ9jTAPYlb4xTd8DOP30ntJUdXPVCxH5FU7zfbnWsiKXiPwTcEZVX6rPgT1vCuS6HfAfXI9vcG5qcwrnB8Ze4J9VdVmN7U8B00RkGXAWuBunpWADzg+CH6nqGte4gwmq+kZ94jPGVGdX+sY0Ya6+/H8Ffn2RzyNFpBPOrWTvAyaISIaI9K5RdDBwc433QkQk2uPRQUQ613KMPq4m+beAf1HVR3H6/iNU9f9c+84BlojIW654ajoNfI0zLuFWnHEC44EuIpKCc2/2RJv2Z8ylsaRvTNP2BU5C3Vnj/arm/VLgCuDnQDpOP/kioHuN8hV49PO7XAGc8HgcAdZ6FhCRvsBfcAbYJavqrzzi+lpEQtWxCOdmIFdS4++OiAwD3gduw2kVWI0zen8VMATnxjjdcGYhrBSRqLoqxRhTO0v6xjRhqlqhqs/X8lGaq4+9JU5S/h/geVUdidOMvsOL3X+uqlL1wBk9X3MdgL1AL1W9S1X31fisHx4zAlR1o+s998wBEZkD/BanJaII525sbYB/w5kG2B/oBaTg3AL3OpwR/8aYBrA+fWOClKspvKo5fCMwVET24PTdH6zv/lS1AqdFwNNa4JbvaHWvvMhnIiIhwH8A/6mq+11v9se5H/jLwKeqOsN1i9aDqvqliCQD6mpBqNkyYYypgyV9Y4LTux7PrwL+C2dVPAWWNOJxhrv26V5cR0R6ALuAEuAtVZ1ZVdg1T79qimASTrfEWRGpudhPK5wfDBke28K3awUMwRnZb4ypB0v6xgSnqsV5woAKVVURWY/TVx7XWAdR1TOer0UkHvg9TpP9PODPrhaHJ1S11LWS31nXtru5yN8gEXkD+ExVZzRWrMYY69M3JliE8u2/57CqN1X1HNBaROYCQ4HdwCsiEuu6CU6yiPTEmfLXVkSuFZFrcebDh1W9dj3+0VU+sebBRSRGRGbiXOEXAY+o6hfAAJy++L0iMl1E2vquCowxdbErfWOCQyTfLlrjXmHPdQOedTgj4lNwRuEvBj7CGRT3PtXX3d9eY781X4e59ne3a/8zcKYCfh/4X2CKqv53VWFXP3wqkImzkM8iEXlNVcfXcT6eP2KMMY1EalmnwxgTREQkVlUP1XivQ9Wyt5e470HA7cAbrub67yobgTNl8AtVfa+OshuAYlXNvNQYjTHfsqRvjDHGNBPWfGaMMcY0E5b0jTHGmGbCkr4xxhjTTFjSN8YYY5oJS/rGGGNMM2FJ3xhjjGkm/h+9S4Y0bPbZ5gAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 576x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(8, 6))\n",
    "plt.plot(fpr, tpr, \"b:\", linewidth=2, label=\"SGD\")\n",
    "plot_roc_curve(fpr_forest, tpr_forest, \"Random Forest\")\n",
    "plt.legend(loc=\"lower right\", fontsize=16)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9924133299476886"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Rand 比SGD 好很多，ROC AUC的分数也高很多\n",
    "roc_auc_score(y_train_5, y_scores_forest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9832608695652174"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 再看一下 精度和召回率 也很高\n",
    "y_train_pred_forest = cross_val_predict(forest_clf, X_train, y_train_5, cv=3)\n",
    "precision_score(y_train_5, y_train_pred_forest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8343479062903524"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "recall_score(y_train_5, y_train_pred_forest)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 总结\n",
    "* 选择合适的指标利用交叉验证来对分类器进行评估\n",
    "* 选择满足需求的精度/召回率权衡\n",
    "* 使用ROC曲线和ROC AUC分数比较多个模型"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 多类别分类器\n",
    "* 尝试5 之外的检测\n",
    "* 多类别分类器 区分两个以上的类别\n",
    "* 随机森林和朴素贝叶斯可以直接处理多个类别\n",
    "* 支持向量机svm和线性分类器只可以处理二元分类器"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 优缺点\n",
    "* OvO 只需要用到部分训练集对其必须区分两个类别进行训练\n",
    "* 对于较小训练集合OvO比较有优势， 大训练集合 OvA 速度快，所以OvA更常用，比如svm 在数据规模扩大时表现糟糕\n",
    "* sklearn 检查到使用二元分类算法进行多类别分类任务，会自动运行OvA，SVM分类器除外"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\86188\\appdata\\local\\programs\\python\\python38\\lib\\site-packages\\sklearn\\linear_model\\stochastic_gradient.py:123: FutureWarning: max_iter and tol parameters have been added in <class 'sklearn.linear_model.stochastic_gradient.SGDClassifier'> in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([5.])"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sgd_clf.fit(X_train, y_train)\n",
    "sgd_clf.predict([some_digit])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ -97796.13245623, -464370.48810561, -338504.99001034,\n",
       "         -73683.22440847, -333467.38328488,   20200.84266059,\n",
       "        -685343.39573847, -366104.60675855, -509726.04105497,\n",
       "        -641986.88514787]])"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 内部实际上训练了10个二元分类器，获得图片的决策分数，然后选择了分数最高的类别\n",
    "# 返回10个分数，每个类别1个\n",
    "some_digit_scores = sgd_clf.decision_function([some_digit])\n",
    "some_digit_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.argmax(some_digit_scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0., 1., 2., 3., 4., 5., 6., 7., 8., 9.])"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 目标类别列表会存储在classes_这个属性中，按值大小排列，\n",
    "sgd_clf.classes_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5.0"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sgd_clf.classes_[np.argmax(some_digit_scores)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([3.])"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 使用OvO策略，一对一或者一对多\n",
    "from sklearn.multiclass import OneVsOneClassifier\n",
    "ovo_clf = OneVsOneClassifier(SGDClassifier(max_iter=5, tol=-np.infty, random_state=42))\n",
    "ovo_clf.fit(X_train, y_train)\n",
    "ovo_clf.predict([some_digit])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "45"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(ovo_clf.estimators_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([5.])"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 使用随机森林\n",
    "forest_clf.fit(X_train, y_train)\n",
    "forest_clf.predict([some_digit])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.1, 0. , 0. , 0. , 0. , 0.7, 0. , 0.1, 0.1, 0. ]])"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 随机森林直接将实例分为多个类别，调用predict_proba()可以获得分类器将每个实例分类为每个类别的概率列表\n",
    "forest_clf.predict_proba([some_digit])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 评估分类器"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\86188\\appdata\\local\\programs\\python\\python38\\lib\\site-packages\\sklearn\\linear_model\\stochastic_gradient.py:123: FutureWarning: max_iter and tol parameters have been added in <class 'sklearn.linear_model.stochastic_gradient.SGDClassifier'> in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n",
      "  warnings.warn(\n",
      "c:\\users\\86188\\appdata\\local\\programs\\python\\python38\\lib\\site-packages\\sklearn\\linear_model\\stochastic_gradient.py:123: FutureWarning: max_iter and tol parameters have been added in <class 'sklearn.linear_model.stochastic_gradient.SGDClassifier'> in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n",
      "  warnings.warn(\n",
      "c:\\users\\86188\\appdata\\local\\programs\\python\\python38\\lib\\site-packages\\sklearn\\linear_model\\stochastic_gradient.py:123: FutureWarning: max_iter and tol parameters have been added in <class 'sklearn.linear_model.stochastic_gradient.SGDClassifier'> in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([0.84533093, 0.82844142, 0.8573286 ])"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 使用交叉验证评估SGD的准确率\n",
    "cross_val_score(sgd_clf, X_train, y_train, cv=3, scoring=\"accuracy\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-69-fd3c7b12f54d>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      2\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0msklearn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpreprocessing\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mStandardScaler\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[0mscaler\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mStandardScaler\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 4\u001b[1;33m \u001b[0mX_train_scaled\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mscaler\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit_transform\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX_train\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mastype\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfloat64\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      5\u001b[0m \u001b[0mcross_val_score\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msgd_clf\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mX_train_scaled\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_train\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcv\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m3\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mscoring\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m\"accuracy\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\86188\\appdata\\local\\programs\\python\\python38\\lib\\site-packages\\sklearn\\base.py\u001b[0m in \u001b[0;36mfit_transform\u001b[1;34m(self, X, y, **fit_params)\u001b[0m\n\u001b[0;32m    515\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0my\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    516\u001b[0m             \u001b[1;31m# fit method of arity 1 (unsupervised transformation)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 517\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mfit_params\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtransform\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    518\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    519\u001b[0m             \u001b[1;31m# fit method of arity 2 (supervised transformation)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\86188\\appdata\\local\\programs\\python\\python38\\lib\\site-packages\\sklearn\\preprocessing\\data.py\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, X, y)\u001b[0m\n\u001b[0;32m    588\u001b[0m         \u001b[1;31m# Reset internal state before fitting\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    589\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_reset\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 590\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpartial_fit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    591\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    592\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mpartial_fit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\86188\\appdata\\local\\programs\\python\\python38\\lib\\site-packages\\sklearn\\preprocessing\\data.py\u001b[0m in \u001b[0;36mpartial_fit\u001b[1;34m(self, X, y)\u001b[0m\n\u001b[0;32m    609\u001b[0m         \u001b[0my\u001b[0m \u001b[1;33m:\u001b[0m \u001b[0mPassthrough\u001b[0m \u001b[1;32mfor\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0mPipeline\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m \u001b[0mcompatibility\u001b[0m\u001b[1;33m.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    610\u001b[0m         \"\"\"\n\u001b[1;32m--> 611\u001b[1;33m         X = check_array(X, accept_sparse=('csr', 'csc'), copy=self.copy,\n\u001b[0m\u001b[0;32m    612\u001b[0m                         warn_on_dtype=True, estimator=self, dtype=FLOAT_DTYPES)\n\u001b[0;32m    613\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\86188\\appdata\\local\\programs\\python\\python38\\lib\\site-packages\\sklearn\\utils\\validation.py\u001b[0m in \u001b[0;36mcheck_array\u001b[1;34m(array, accept_sparse, dtype, order, copy, force_all_finite, ensure_2d, allow_nd, ensure_min_samples, ensure_min_features, warn_on_dtype, estimator)\u001b[0m\n\u001b[0;32m    442\u001b[0m             \u001b[0marray\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0matleast_2d\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0marray\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    443\u001b[0m             \u001b[1;31m# To ensure that array flags are maintained\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 444\u001b[1;33m             \u001b[0marray\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0marray\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0marray\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mdtype\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0morder\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0morder\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcopy\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mcopy\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    445\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    446\u001b[0m         \u001b[1;31m# make sure we actually converted to numeric:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "# 将输入进行简单缩放 ，可以得到准确率 90 %以上\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "scaler = StandardScaler()\n",
    "X_train_scaled = scaler.fit_transform(X_train.astype(np.float64))\n",
    "cross_val_score(sgd_clf, X_train_scaled, y_train, cv=3, scoring=\"accuracy\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 错误分析"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 项目流程\n",
    "1. 探索数据准备的选项\n",
    "2. 尝试多个模型\n",
    "3. 选择最佳模型并用GridSearchCV对参数进行微调\n",
    "4. 尽可能自动化\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 确定了一个相对合适的模型，进一步优化，分析其错误类型\n",
    "* 查看混淆矩阵\n",
    "* 使用cross_val_predict()进行预测\n",
    "* 调用confusion_matrix()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train_pred = cross_val_predict(sgd_clf, X_train_scaled, y_train, cv=3)\n",
    "conf_mx = confusion_matrix(y_train, y_train_pred)\n",
    "conf_mx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 使用matplotlib的matshow 函数来查看混淆矩阵的图像表示\n",
    "plt.matshow(conf_mx, cmap = plt.cm.gray)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 看起来不错，大多数图片都在主对角线上，说明它们被正确分类\n",
    "# 数字5 看起来比较暗，说明1. 数字5图片较少  2. 分类器在数字5上执行效果不如其他数字上好\n",
    "# 假设把焦点放在错误上，为取得错误率，而不是错误绝对值，需要将混淆矩阵中每个值除以相应类别中的图片数量\n",
    "\n",
    "row_sums = conf_mx.sum(axis=1, keepdims=True)\n",
    "norm_conf_mx = conf_mx / row_sums\n",
    "norm_conf_mx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 用0填充对角线 只保留错误，重新绘制\n",
    "np.fill_diagonal(norm_conf_mx, 0)\n",
    "plt.matshow(norm_conf_mx, cmap=plt.cm.gray)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 每行代表实际类别，每列代表预测类别\n",
    "# 8，9列比较亮，说明许多图片被错误的分类为数字8，9\n",
    "# 类别8，9行也偏亮，说明数字8和9经常会跟其他数字混淆\n",
    "# 大多数数字1都被正确的分类有些很暗，比如行1，，一些和8混淆\n",
    "# 5和3是错误最多的"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 结论\n",
    "* 改进数字8和9的分类\n",
    "* 修正数字3和5的混淆\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 如何优化分类器\n",
    "* 尝试多收集这些数字的训练集\n",
    "* 开发一些新特征来改进分类器\n",
    "* 优化分类器算法\n",
    "* 使用pillow或opencv对图片预处理，让显示模型更突出\n",
    "* 分析单个错误"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_digits(instances, images_per_row=10, **options):\n",
    "    size = 28\n",
    "    images_per_row = min(len(instances), images_per_row)\n",
    "    images = [instance.reshape(size,size) for instance in instances]\n",
    "    n_rows = (len(instances) - 1) // images_per_row + 1\n",
    "    row_images = []\n",
    "    n_empty = n_rows * images_per_row - len(instances)\n",
    "    images.append(np.zeros((size, size * n_empty)))\n",
    "    for row in range(n_rows):\n",
    "        rimages = images[row * images_per_row : (row + 1) * images_per_row]\n",
    "        row_images.append(np.concatenate(rimages, axis=1))\n",
    "    image = np.concatenate(row_images, axis=0)\n",
    "    plt.imshow(image, cmap = matplotlib.cm.binary, **options)\n",
    "    plt.axis(\"off\")\n",
    "\n",
    "\n",
    "# 查看数字3和数字5的例子\n",
    "cl_a, cl_b = 3, 5\n",
    "X_aa = X_train[(y_train == cl_a) & (y_train_pred == cl_a)]\n",
    "X_ab = X_train[(y_train == cl_a) & (y_train_pred == cl_b)]\n",
    "X_ba = X_train[(y_train == cl_b) & (y_train_pred == cl_a)]\n",
    "X_bb = X_train[(y_train == cl_b) & (y_train_pred == cl_b)]\n",
    "\n",
    "plt.figure(figsize=(8,8))\n",
    "plt.subplot(221); \n",
    "plot_digits(X_aa[:25], images_per_row=5)\n",
    "plt.subplot(222); \n",
    "plot_digits(X_ab[:25], images_per_row=5)\n",
    "plt.subplot(223);\n",
    "plot_digits(X_ba[:25], images_per_row=5)\n",
    "plt.subplot(224); \n",
    "plot_digits(X_bb[:25], images_per_row=5)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 左侧两个是被分类为3的图片\n",
    "# 右侧两个是被分类为5的图片\n",
    "# 大多数错误分类的图片看起来还是非常明显的错误\n",
    "# 原因：SGD是一个线性模型，它所做就是为每个像素分配一个各个类别的权重，当它看到新的图像，将加权后的像素强度汇总，从而得到一个分数进行分类\n",
    "# 数字3和5在一部分像素位上有区别，所以分类器很容易将其弄混\n",
    "# 通过上面图像，如果书写3 的连接点左移，分类器可能将其分类为数字5，这个分类器对图像位移和旋转敏感\n",
    "# 减少混淆的方法之一，就是对图像进行预处理，确保位于中心位置并且没有旋转"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 多标签分类"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 为每个实例产生多个类别 ，例如 照片识别多个人脸\n",
    "# 分类器经过训练可以识别小红，小白，小军，一张照片 里有 小红，小白\n",
    "# 经过分类器，应该输出[1,1,0]， 是小红，是小白，不是小军\n",
    "# 输出多个二元标签的分类系统称为多标签分类系统"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "\n",
    "y_train_large = (y_train >= 7)\n",
    "y_train_odd = (y_train % 2 == 1)\n",
    "y_multilabel = np.c_[y_train_large, y_train_odd]\n",
    "\n",
    "knn_clf = KNeighborsClassifier()\n",
    "knn_clf.fit(X_train, y_multilabel)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# knn支持多标签分类，不是所有的分类器都支持\n",
    "knn_clf.predict([some_digit])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # 评估多标签分类器方法很多，方法之一就是测量每个标签的F1分数，或者其他二元分类器指标，然后简单平均\n",
    "# y_train_knn_pred = cross_val_predict(knn_clf, X_train, y_multilabel, cv=3)\n",
    "# f1_score(y_multilabel, y_train_knn_pred, average=\"macro\")\n",
    "0.977090"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 上面假设了所有标签都同等重要，也可以给每个标签设置一个权重（该目标标签实例的数量），设置average='weighted'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 多输出分类"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 例子：构建一个系统去除图片中的噪声，输入一张有噪声的图片，它将输入一张干净的数字图片，分类器输出是多个标签，一个像素一个标签，每个标签多个值0到255"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 增加噪声，目标将图片还原为原始图片 创建训练集和测试集\n",
    "noise = np.random.randint(0, 100, (len(X_train), 784))\n",
    "X_train_mod = X_train + noise\n",
    "noise = np.random.randint(0, 100, (len(X_test), 784))\n",
    "X_test_mod = X_test + noise\n",
    "y_train_mod = X_train\n",
    "y_test_mod = X_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "some_index = 5500\n",
    "plt.subplot(121);plt.imshow(X_test_mod[some_index].reshape(28, 28), cmap = matplotlib.cm.binary)\n",
    "plt.subplot(122);plt.imshow(y_test_mod[some_index].reshape(28, 28), cmap = matplotlib.cm.binary)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "knn_clf.fit(X_train_mod, y_train_mod)\n",
    "clean_digit = knn_clf.predict([X_test_mod[some_index]])\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.imshow(clean_digit.reshape(28, 28), cmap = matplotlib.cm.binary)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 垃圾邮件分类"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "SPAM_PATH = os.path.join(\"datasets\", \"spam\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 获取垃圾邮件和正常邮件的路径和名称\n",
    "HAM_DIR = os.path.join(SPAM_PATH, \"easy_ham\")\n",
    "SPAM_DIR = os.path.join(SPAM_PATH, \"spam\")\n",
    "ham_filenames = [name for name in sorted(os.listdir(HAM_DIR)) if len(name) > 20]\n",
    "spam_filenames = [name for name in sorted(os.listdir(SPAM_DIR)) if len(name) > 20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 我们可以使用python的“email”模块解析这些电子邮件（它处理邮件头、编码等）\n",
    "import email\n",
    "import email.policy\n",
    "\n",
    "def load_email(is_spam, filename, spam_path=SPAM_PATH):\n",
    "    directory = \"spam\" if is_spam else \"easy_ham\"\n",
    "    with open(os.path.join(spam_path, directory, filename), \"rb\") as f:\n",
    "        return email.parser.BytesParser(policy=email.policy.default).parse(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 让我们看一个ham示例和一个spam示例，了解数据的外观：\n",
    "ham_emails = [load_email(is_spam=False, filename=name) for name in ham_filenames]\n",
    "spam_emails = [load_email(is_spam=True, filename=name) for name in spam_filenames]\n",
    "print(ham_emails[1].get_content().strip())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(spam_emails[6].get_content().strip())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 电子邮件实际上有很多部分，带有图像和附件（它们可以有自己的附件）。查看邮件的各种类型的结构：\n",
    "def get_email_structure(email):\n",
    "    if isinstance(email, str):\n",
    "        return email\n",
    "    payload = email.get_payload()\n",
    "    if isinstance(payload, list):\n",
    "        return \"multipart({})\".format(\", \".join([\n",
    "            get_email_structure(sub_email)\n",
    "            for sub_email in payload\n",
    "        ]))\n",
    "    else:\n",
    "        return email.get_content_type()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "\n",
    "a = [1,4,2,3,2,3,4,2]  \n",
    " \n",
    "b = Counter(a) #求数组中每个数字出现了几次\n",
    "b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def structures_counter(emails):\n",
    "    structures = Counter()\n",
    "    for email in emails:\n",
    "        structure = get_email_structure(email)\n",
    "        structures[structure] += 1\n",
    "    return structures"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "structures_counter(ham_emails).most_common()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "structures_counter(spam_emails).most_common()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 正常邮件更多的是纯文本，而垃圾邮件有相当多的HTML。\n",
    "# 查看邮件头\n",
    "for header, value in spam_emails[0].items():\n",
    "    print(header,\":\",value)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 里面可能有很多有用的信息，比如发件人的电子邮件地址（12a1mailbot1@web.de看起来很可疑），\n",
    "# 查看“主题”标题：\n",
    "\n",
    "spam_emails[0][\"Subject\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 拆分训练集和测试集合"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "X = np.array(ham_emails + spam_emails)\n",
    "y = np.array([0] * len(ham_emails) + [1] * len(spam_emails))\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 首先需要一个函数来将html转换为纯文本，使用[Beautifulsoup]库，下面的函数首先删除`<head>`部分，然后将所有`<a>`标记转换为单词hyperlink，然后去掉所有html标记，只留下纯文本。为了可读性，它还用一个换行符替换多个换行符，最后它取消了HTML实体（例如`&gt；`或`&nbsp；`）\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "from html import unescape\n",
    "\n",
    "def html_to_plain_text(html):\n",
    "    text = re.sub('<head.*?>.*?</head>', '', html, flags=re.M | re.S | re.I)\n",
    "    text = re.sub('<a\\s.*?>', ' HYPERLINK ', text, flags=re.M | re.S | re.I)\n",
    "    text = re.sub('<.*?>', '', text, flags=re.M | re.S)\n",
    "    text = re.sub(r'(\\s*\\n)+', '\\n', text, flags=re.M | re.S)\n",
    "    return unescape(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "html_spam_emails = [email for email in X_train[y_train==1]\n",
    "                    if get_email_structure(email) == \"text/html\"]\n",
    "sample_html_spam = html_spam_emails[7]\n",
    "print(sample_html_spam.get_content().strip()[:1000], \"...\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(html_to_plain_text(sample_html_spam.get_content())[:1000], \"...\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 编写一个函数，它以电子邮件为输入，并以纯文本形式返回其内容，无论其格式是什么\n",
    "def email_to_text(email):\n",
    "    html = None\n",
    "    for part in email.walk():\n",
    "        ctype = part.get_content_type()\n",
    "        if not ctype in (\"text/plain\", \"text/html\"):\n",
    "            continue\n",
    "        try:\n",
    "            content = part.get_content()\n",
    "        except: # 解决编码问题\n",
    "            content = str(part.get_payload())\n",
    "        if ctype == \"text/plain\":\n",
    "            return content\n",
    "        else:\n",
    "            html = content\n",
    "    if html:\n",
    "        return html_to_plain_text(html)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(email_to_text(sample_html_spam)[:100], \"...\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 装自然语言工具包（[nltk]（http://www.nltk.org/）\n",
    "# pip install nltk\n",
    "\n",
    "# 用“url”替换url的方法 \n",
    "# pip install urlextract\n",
    "import nltk\n",
    "from urlextract import URLExtract\n",
    "\n",
    "try:\n",
    "    import nltk\n",
    "\n",
    "    stemmer = nltk.PorterStemmer()\n",
    "    for word in (\"Computations\", \"Computation\", \"Computing\", \"Computed\", \"Compute\", \"Compulsive\"):\n",
    "        print(word, \"=>\", stemmer.stem(word))\n",
    "except ImportError:\n",
    "    print(\"Error: stemming requires the NLTK module.\")\n",
    "    stemmer = None\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 将所有处理整合到一个转换器中，我们将使用它将电子邮件转换为文字计数器。注意，我们使用python的'split（）'方法将句子拆分为单词，该方法使用空格作为单词边界。但例如，汉语和日语脚本通常不在单词之间使用空格在这个练习中没关系，因为数据集（主要）是英文的，中文可以使用结巴分词来进行拆分"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.base import BaseEstimator, TransformerMixin\n",
    "\n",
    "class EmailToWordCounterTransformer(BaseEstimator, TransformerMixin):\n",
    "    def __init__(self, strip_headers=True, lower_case=True, remove_punctuation=True,\n",
    "                 replace_urls=True, replace_numbers=True,  stemming=True):\n",
    "        self.strip_headers = strip_headers\n",
    "        self.lower_case = lower_case\n",
    "        self.remove_punctuation = remove_punctuation\n",
    "        self.replace_urls = replace_urls\n",
    "        self.replace_numbers = replace_numbers\n",
    "        self.stemming = stemming\n",
    "    def fit(self, X, y=None):\n",
    "        return self\n",
    "    def transform(self, X, y=None):\n",
    "        X_transformed = []\n",
    "        for email in X:\n",
    "            text = email_to_text(email) or \"\"\n",
    "            if self.lower_case:\n",
    "                text = text.lower()\n",
    "            if self.replace_urls:\n",
    "                extractor = URLExtract()\n",
    "                urls = list(set(extractor.find_urls(text)))\n",
    "                urls.sort(key=lambda url: len(url), reverse=True)\n",
    "                for url in urls:  # 替换url 为 ‘URL’\n",
    "                    text = text.replace(url, \" URL \")\n",
    "            if self.replace_numbers:  # 替换数字\n",
    "                text = re.sub(r'\\d+(?:\\.\\d*(?:[eE]\\d+))?', 'NUMBER', text)\n",
    "            if self.remove_punctuation:  # 删除标点符号\n",
    "                text = re.sub(r'\\W+', ' ', text, flags=re.M)\n",
    "            word_counts = Counter(text.split())\n",
    "            if self.stemming and stemmer is not None:\n",
    "                stemmed_word_counts = Counter()\n",
    "                for word, count in word_counts.items():\n",
    "                    stemmed_word = stemmer.stem(word)\n",
    "                    stemmed_word_counts[stemmed_word] += count\n",
    "                word_counts = stemmed_word_counts\n",
    "            X_transformed.append(word_counts)\n",
    "        return np.array(X_transformed)\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# 在一些邮件上 测试 转换器\n",
    "X_few = X_train[:3]\n",
    "X_few_wordcounts = EmailToWordCounterTransformer().fit_transform(X_few)\n",
    "X_few_wordcounts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.sparse import csr_matrix\n",
    "\n",
    "class WordCounterToVectorTransformer(BaseEstimator, TransformerMixin):\n",
    "    def __init__(self, vocabulary_size = 1000):\n",
    "        self.vocabulary_size = vocabulary_size  # 词汇量\n",
    "    def fit(self, X, y = None):\n",
    "        total_count = Counter()\n",
    "        for word_count in X:\n",
    "            for word, count in word_count.items():\n",
    "                total_count[word] += min(count, 10)\n",
    "        most_common = total_count.most_common()[:self.vocabulary_size]\n",
    "        self.most_common_ = most_common\n",
    "        self.vocabulary_ = {word: index + 1 for index, (word, count) in enumerate(most_common)}\n",
    "        return self\n",
    "    def transform(self, X, y = None):\n",
    "        rows = []\n",
    "        cols = []\n",
    "        data = []\n",
    "        for row, word_count in enumerate(X):\n",
    "            for word, count in word_count.items():\n",
    "                rows.append(row) # 训练集 实例个数\n",
    "                cols.append(self.vocabulary_.get(word, 0)) # 取得单词在词汇表中的索引位置，0代表未出现在词汇表中\n",
    "                data.append(count)\n",
    "        return csr_matrix((data, (rows, cols)), shape=(len(X), self.vocabulary_size + 1)) # 输出稀疏矩阵 +1因为第一列要显示未出现在词汇表中的单词统计数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# from scipy.sparse import *\n",
    " \n",
    "# row =  [0,0,0,1,1,1,2,2,2]#行指标\n",
    "# col =  [0,1,2,0,1,2,0,1,2]#列指标\n",
    "# data = [1,0,1,0,1,1,1,1,0]#在行指标列指标下的数字\n",
    "# team = csr_matrix((data,(row,col)),shape=(3,3))\n",
    "# print(team)\n",
    "# print(team.todense())\n",
    "# team.toarray()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "vocab_transformer = WordCounterToVectorTransformer(vocabulary_size=10)\n",
    "X_few_vectors = vocab_transformer.fit_transform(X_few_wordcounts)\n",
    "X_few_vectors.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_few_vectors.toarray()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 第三行第一列中的67表示第三封电子邮件包含64个不属于词汇表的单词。旁边的1表示词汇表中'of'单词在此电子邮件中出现一次。旁边的2表示'and'单词出现两次,'the'没有出现"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "vocab_transformer.vocabulary_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 我们现在准备训练我们的第一个垃圾邮件分类器！让我们转换整个数据集："
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.pipeline import Pipeline\n",
    "\n",
    "preprocess_pipeline = Pipeline([\n",
    "    (\"email_to_wordcount\", EmailToWordCounterTransformer()),\n",
    "    (\"wordcount_to_vector\", WordCounterToVectorTransformer()),\n",
    "])\n",
    "\n",
    "X_train_transformed = preprocess_pipeline.fit_transform(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.model_selection import cross_val_score\n",
    "\n",
    "log_clf = LogisticRegression(solver=\"liblinear\", random_state=42) # 采用逻辑回归分类器\n",
    "score = cross_val_score(log_clf, X_train_transformed, y_train, cv=3, verbose=3)\n",
    "score.mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 得到分数超过98.7%，可以尝试多个模型，选择最好的模型，并使用交叉验证对它们进行微调。在测试集上得到的精度/召回率：\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import precision_score, recall_score\n",
    "\n",
    "X_test_transformed = preprocess_pipeline.transform(X_test)\n",
    "\n",
    "log_clf = LogisticRegression(solver=\"liblinear\", random_state=42)\n",
    "log_clf.fit(X_train_transformed, y_train)\n",
    "\n",
    "y_pred = log_clf.predict(X_test_transformed)\n",
    "\n",
    "print(\"精度: {:.2f}%\".format(100 * precision_score(y_test, y_pred)))\n",
    "print(\"召回: {:.2f}%\".format(100 * recall_score(y_test, y_pred)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 总结\n",
    "1. 加载数据并纵观数据大局\n",
    "2. 获取邮件的组成结构\n",
    "3. 对结构类型进行分析 发现垃圾邮件大多有HTML结构\n",
    "4. 数据清洗，定义email对象中的HTML转换称纯文本方法\n",
    "5. 对数据集拆分成训练集和测试集\n",
    "6. 数据处理转换，对邮件的文本内容进行分词处理，通过nltk进行词干提取，对邮件出现的词汇进行计数统计，对所有邮件统计出了一个词汇表\n",
    "7. 通过词汇表和邮件单词计数统计，将单词计数转化成向量矩阵\n",
    "8. 把数据清洗和数据处理封装成两个转换器\n",
    "9. 通过流水线来自动化处理数据\n",
    "10. 使用逻辑回归线性分类器进行模型训练\n",
    "11. 使用交叉验证进行微调\n",
    "12. 在测试集上得到精度/召回率"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
